From ac33a1a540b902f7a6e1f0d1a936c3edf96c9110 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Mon, 22 Jul 2024 15:37:32 -0400 Subject: [PATCH 01/21] Add SpecialCaseBinaryStringSchema --- examples/codec.ts | 2 +- package-lock.json | 7 +- package.json | 3 +- src/encoding/binarydata.ts | 20 ++- src/encoding/encoding.ts | 197 ++++++++++++++++++++++++++-- src/encoding/schema/address.ts | 13 +- src/encoding/schema/array.ts | 19 ++- src/encoding/schema/binarystring.ts | 59 +++++++++ src/encoding/schema/boolean.ts | 13 +- src/encoding/schema/bytearray.ts | 19 ++- src/encoding/schema/index.ts | 2 + src/encoding/schema/map.ts | 37 +++++- src/encoding/schema/optional.ts | 14 +- src/encoding/schema/string.ts | 13 +- src/encoding/schema/uint64.ts | 13 +- src/encoding/schema/untyped.ts | 5 +- src/main.ts | 3 +- tests/2.Encoding.ts | 102 ++++++++++++-- 18 files changed, 479 insertions(+), 62 deletions(-) create mode 100644 src/encoding/schema/binarystring.ts diff --git a/examples/codec.ts b/examples/codec.ts index e92f7f4d9..3ba74c7ed 100644 --- a/examples/codec.ts +++ b/examples/codec.ts @@ -20,7 +20,7 @@ async function main() { // example: CODEC_BASE64 const b64Encoded = 'SGksIEknbSBkZWNvZGVkIGZyb20gYmFzZTY0'; - const b64Decoded = algosdk.base64ToString(b64Encoded); + const b64Decoded = algosdk.base64ToBytes(b64Encoded); console.log(b64Encoded, b64Decoded); // example: CODEC_BASE64 diff --git a/package-lock.json b/package-lock.json index 513d54a03..fcc2372f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "2.8.0", "license": "MIT", "dependencies": { - "algorand-msgpack": "^1.0.1", + "algorand-msgpack": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", @@ -1196,8 +1196,9 @@ }, "node_modules/algorand-msgpack": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/algorand-msgpack/-/algorand-msgpack-1.0.1.tgz", - "integrity": "sha512-8vVEar8APmTT7cp6Ye2pJg7tHFSgXfJlXmq85v3MTlqPQp14KnxndbyKIq5xjCBT/oarx9p5ncDfRq3hzYg/NA==", + "resolved": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", + "integrity": "sha512-lqopii/JLsD1EmKpOG0Qq3itPzhb7xBkR3P7ClNJicpld06oeQMud3YnmntjDmqxMl/r38Q2T+cFXFdEg6P9xw==", + "license": "ISC", "engines": { "node": ">= 14" } diff --git a/package.json b/package.json index 92f792b46..c991c8c9c 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "url": "git://github.com/algorand/js-algorand-sdk.git" }, "dependencies": { - "algorand-msgpack": "^1.0.1", + "algorand-msgpack": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", @@ -93,6 +93,7 @@ "webpack-cli": "^5.0.1" }, "scripts": { + "preinstall": "cd .. && git clone https://github.com/algorand/msgpack-javascript.git && git checkout raw-string-encoding && cd msgpack-javascript && npm i && npm run build && npm pack", "test": "tsx tests/mocha.js", "prepare": "npm run build", "prepare-browser-tests": "npm run build && mkdir -p tests/cucumber/browser/build && cp dist/browser/algosdk.min.* tests/cucumber/browser/build/ && webpack --config tests/cucumber/browser/webpack.config.js", diff --git a/src/encoding/binarydata.ts b/src/encoding/binarydata.ts index add1f4980..7fd09f457 100644 --- a/src/encoding/binarydata.ts +++ b/src/encoding/binarydata.ts @@ -13,18 +13,6 @@ export function base64ToBytes(base64String: string): Uint8Array { return Uint8Array.from(binString, (m) => m.codePointAt(0)!); } -/** - * Decode a base64 string for Node.js and browser environments. - * @returns A decoded string - */ -export function base64ToString(base64String: string): string { - if (isNode()) { - return Buffer.from(base64String, 'base64').toString(); - } - const binString = base64ToBytes(base64String); - return new TextDecoder().decode(binString); -} - /** * Convert a Uint8Array to a base64 string for Node.js and browser environments. * @returns A base64 string @@ -40,6 +28,14 @@ export function bytesToBase64(byteArray: Uint8Array): string { return btoa(binString); } +/** + * Decode a base64 string for Node.js and browser environments. + * @returns A decoded string + */ +export function bytesToString(byteArray: Uint8Array): string { + return new TextDecoder().decode(byteArray); +} + /** * Returns a Uint8Array given an input string or Uint8Array. * @returns A base64 string diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 5e6a59466..cfb1b0b0e 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -16,10 +16,11 @@ import { decode as msgpackDecode, DecoderOptions, IntMode, + RawBinaryString, } from 'algorand-msgpack'; -import { bytesToBase64 } from './binarydata.js'; +import { bytesToBase64, coerceToBytes } from './binarydata.js'; import IntDecoding from '../types/intDecoding.js'; -import { stringifyJSON, parseJSON } from '../utils/utils.js'; +import { stringifyJSON, parseJSON, arrayEqual } from '../utils/utils.js'; // Errors export const ERROR_CONTAINS_EMPTY_STRING = @@ -72,17 +73,29 @@ export function encode(obj: Record) { } export function decode(buffer: ArrayLike) { - // TODO: consider different int mode + // TODO: make IntMode an argument const options: DecoderOptions = { intMode: IntMode.MIXED }; return msgpackDecode(buffer, options); } export function decodeAsMap(encoded: ArrayLike) { - // TODO: consider different int mode + // TODO: make IntMode an argument const options: DecoderOptions = { intMode: IntMode.MIXED, useMap: true }; return msgpackDecode(encoded, options); } +function decodeAsMapWithRawStrings(encoded: ArrayLike) { + // TODO: make IntMode an argument + const options: DecoderOptions = { + intMode: IntMode.BIGINT, + useMap: true, + rawBinaryStringKeys: true, + rawBinaryStringValues: true, + useRawBinaryStringClass: true, + }; + return msgpackDecode(encoded, options); +} + export type MsgpackEncodingData = | null | undefined @@ -159,6 +172,168 @@ export function jsonEncodingDataToMsgpackEncodingData( /* eslint-disable class-methods-use-this */ /* eslint-disable no-useless-constructor,no-empty-function */ +enum MsgpackObjectPathSegmentKind { + MAP_VALUE, + ARRAY_ELEMENT, +} + +interface MsgpackObjectPathSegment { + kind: MsgpackObjectPathSegmentKind; + key: string | number | bigint | Uint8Array; +} + +export class MsgpackRawStringProvider { + // eslint-disable-next-line no-use-before-define + private readonly parent?: MsgpackRawStringProvider; + + private readonly baseObjectBytes?: ArrayLike; + + private readonly segment?; + + private resolvedCache: MsgpackEncodingData = null; + private resolvedCachePresent = false; + + public constructor({ + parent, + segment, + baseObjectBytes, + }: + | { + parent: MsgpackRawStringProvider; + segment: MsgpackObjectPathSegment; + baseObjectBytes?: undefined; + } + | { + parent?: undefined; + segment?: undefined; + baseObjectBytes: ArrayLike; + }) { + this.parent = parent; + this.segment = segment; + this.baseObjectBytes = baseObjectBytes; + } + + public withMapValue( + key: string | number | bigint | Uint8Array + ): MsgpackRawStringProvider { + return new MsgpackRawStringProvider({ + parent: this, + segment: { + kind: MsgpackObjectPathSegmentKind.MAP_VALUE, + key, + }, + }); + } + + public withArrayElement(index: number): MsgpackRawStringProvider { + return new MsgpackRawStringProvider({ + parent: this, + segment: { + kind: MsgpackObjectPathSegmentKind.ARRAY_ELEMENT, + key: index, + }, + }); + } + + public getRawStringAtCurrentLocation(): Uint8Array { + const resolved = this.resolve(); + if (resolved instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + return resolved.rawBinaryValue as Uint8Array; + } + throw new Error( + `Invalid type. Expected RawBinaryString, got ${resolved} (${typeof resolved})` + ); + } + + public getRawStringKeysAtCurrentLocation(): Uint8Array[] { + const resolved = this.resolve(); + if (!(resolved instanceof Map)) { + throw new Error( + `Invalid type. Expected Map, got ${resolved} (${typeof resolved})` + ); + } + const keys: Uint8Array[] = []; + for (const key of resolved.keys()) { + if (key instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + keys.push(key.rawBinaryValue as Uint8Array); + } else { + throw new Error( + `Invalid type for map key. Expected RawBinaryString, got (${typeof key}) ${key}` + ); + } + } + return keys; + } + + private resolve(): MsgpackEncodingData { + if (this.resolvedCachePresent) { + return this.resolvedCache; + } + let parentResolved: MsgpackEncodingData; + if (this.parent) { + parentResolved = this.parent.resolve(); + } else { + // Need to parse baseObjectBytes + parentResolved = decodeAsMapWithRawStrings( + this.baseObjectBytes! + ) as MsgpackEncodingData; + } + if (!this.segment) { + this.resolvedCache = parentResolved; + this.resolvedCachePresent = true; + return parentResolved; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.MAP_VALUE) { + if (!(parentResolved instanceof Map)) { + throw new Error( + `Invalid type. Expected Map, got ${parentResolved} (${typeof parentResolved})` + ); + } + // All decoded map keys will be raw strings, and Map objects compare complex values by reference, + // so we must check all the values for value-equality. + if ( + typeof this.segment.key === 'string' || + this.segment.key instanceof Uint8Array + ) { + const targetBytes = coerceToBytes(this.segment.key); + const targetIsRawString = typeof this.segment.key === 'string'; + for (const [key, value] of parentResolved) { + let potentialKeyBytes: Uint8Array | undefined; + if (targetIsRawString) { + if (key instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + potentialKeyBytes = key.rawBinaryValue as Uint8Array; + } + } else if (key instanceof Uint8Array) { + potentialKeyBytes = key; + } + if (potentialKeyBytes && arrayEqual(targetBytes, potentialKeyBytes)) { + this.resolvedCache = value; + break; + } + } + } else { + this.resolvedCache = parentResolved.get(this.segment.key); + } + this.resolvedCachePresent = true; + return this.resolvedCache; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.ARRAY_ELEMENT) { + if (!Array.isArray(parentResolved)) { + throw new Error( + `Invalid type. Expected Array, got ${parentResolved} (${typeof parentResolved})` + ); + } + this.resolvedCache = parentResolved[this.segment.key as number]; + this.resolvedCachePresent = true; + return this.resolvedCache; + } + throw new Error(`Invalid segment kind: ${this.segment.kind}`); + } +} + /** * A Schema is used to prepare objects for encoding and decoding from msgpack and JSON. * @@ -187,9 +362,13 @@ export abstract class Schema { /** * Restores the encoding data from a msgpack encoding object. * @param encoded - The msgpack encoding object to restore. + * @param rawStringProvider - A provider for raw strings. * @returns The original encoding data. */ - public abstract fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown; + public abstract fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown; /** * Prepares the encoding data for encoding to JSON. @@ -246,10 +425,12 @@ export function decodeMsgpack( encoded: ArrayLike, c: EncodableClass ): T { + const decoded = decodeAsMap(encoded) as MsgpackEncodingData; + const rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: encoded, + }); return c.fromEncodingData( - c.encodingSchema.fromPreparedMsgpack( - decodeAsMap(encoded) as MsgpackEncodingData - ) + c.encodingSchema.fromPreparedMsgpack(decoded, rawStringProvider) ); } diff --git a/src/encoding/schema/address.ts b/src/encoding/schema/address.ts index 73da74365..6de59fb1b 100644 --- a/src/encoding/schema/address.ts +++ b/src/encoding/schema/address.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; import { Address } from '../address.js'; /* eslint-disable class-methods-use-this */ @@ -20,7 +25,11 @@ export class AddressSchema extends Schema { throw new Error(`Invalid address: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Address { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Address { // The Address constructor checks that the input is a Uint8Array return new Address(encoded as Uint8Array); } diff --git a/src/encoding/schema/array.ts b/src/encoding/schema/array.ts index 9008b3b93..c47995dc7 100644 --- a/src/encoding/schema/array.ts +++ b/src/encoding/schema/array.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -22,9 +27,17 @@ export class ArraySchema extends Schema { throw new Error('ArraySchema data must be an array'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown[] { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown[] { if (Array.isArray(encoded)) { - return encoded.map((item) => this.itemSchema.fromPreparedMsgpack(item)); + return encoded.map((item, index) => + this.itemSchema.fromPreparedMsgpack( + item, + rawStringProvider.withArrayElement(index) + ) + ); } throw new Error('ArraySchema encoded data must be an array'); } diff --git a/src/encoding/schema/binarystring.ts b/src/encoding/schema/binarystring.ts new file mode 100644 index 000000000..a7d3c37a8 --- /dev/null +++ b/src/encoding/schema/binarystring.ts @@ -0,0 +1,59 @@ +import { RawBinaryString } from 'algorand-msgpack'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; +import { coerceToBytes, bytesToString } from '../binarydata.js'; + +/* eslint-disable class-methods-use-this */ + +/** + * SpecialCaseBinaryStringSchema is a schema for byte arrays which are encoded + * as strings in msgpack and JSON. + * + * This schema allows lossless conversion between the in memory representation + * and the msgpack encoded representation, but NOT between the in memory and + * JSON encoded representations if the byte array contains invalid UTF-8 + * sequences. + */ +export class SpecialCaseBinaryStringSchema extends Schema { + public defaultValue(): Uint8Array { + return new Uint8Array(); + } + + public isDefaultValue(data: unknown): boolean { + return data instanceof Uint8Array && data.byteLength === 0; + } + + public prepareMsgpack(data: unknown): MsgpackEncodingData { + if (data instanceof Uint8Array) { + // TODO: fix cast? + return new RawBinaryString(data) as unknown as MsgpackEncodingData; + } + throw new Error(`Invalid byte array: (${typeof data}) ${data}`); + } + + public fromPreparedMsgpack( + _encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { + return rawStringProvider.getRawStringAtCurrentLocation(); + } + + public prepareJSON(data: unknown): JSONEncodingData { + if (data instanceof Uint8Array) { + // WARNING: not safe for all binary data + return bytesToString(data); + } + throw new Error(`Invalid byte array: (${typeof data}) ${data}`); + } + + public fromPreparedJSON(encoded: JSONEncodingData): Uint8Array { + if (typeof encoded === 'string') { + return coerceToBytes(encoded); + } + throw new Error(`Invalid byte array: (${typeof encoded}) ${encoded}`); + } +} diff --git a/src/encoding/schema/boolean.ts b/src/encoding/schema/boolean.ts index a5ebff584..17bb53475 100644 --- a/src/encoding/schema/boolean.ts +++ b/src/encoding/schema/boolean.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -18,7 +23,11 @@ export class BooleanSchema extends Schema { throw new Error('Invalid boolean'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): boolean { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): boolean { if (typeof encoded === 'boolean') { return encoded; } diff --git a/src/encoding/schema/bytearray.ts b/src/encoding/schema/bytearray.ts index 0cffc0f75..a063203d0 100644 --- a/src/encoding/schema/bytearray.ts +++ b/src/encoding/schema/bytearray.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; import { base64ToBytes, bytesToBase64 } from '../binarydata.js'; /* eslint-disable class-methods-use-this */ @@ -19,7 +24,11 @@ export class ByteArraySchema extends Schema { throw new Error(`Invalid byte array: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Uint8Array { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { if (encoded instanceof Uint8Array) { return encoded; } @@ -70,7 +79,11 @@ export class FixedLengthByteArraySchema extends Schema { throw new Error('Invalid byte array'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Uint8Array { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { if (encoded instanceof Uint8Array) { if (encoded.byteLength === this.length) { return encoded; diff --git a/src/encoding/schema/index.ts b/src/encoding/schema/index.ts index 01077572c..2ae03c2c5 100644 --- a/src/encoding/schema/index.ts +++ b/src/encoding/schema/index.ts @@ -5,6 +5,8 @@ export { Uint64Schema } from './uint64.js'; export { AddressSchema } from './address.js'; export { ByteArraySchema, FixedLengthByteArraySchema } from './bytearray.js'; +export { SpecialCaseBinaryStringSchema } from './binarystring.js'; + export { ArraySchema } from './array.js'; export { NamedMapSchema, diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 21984f06b..7ce968e1f 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; import { ensureUint64 } from '../../utils/utils.js'; /* eslint-disable class-methods-use-this */ @@ -139,7 +144,8 @@ export class NamedMapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('NamedMapSchema data must be a Map'); @@ -149,7 +155,10 @@ export class NamedMapSchema extends Schema { if (encoded.has(entry.key)) { map.set( entry.key, - entry.valueSchema.fromPreparedMsgpack(encoded.get(entry.key)) + entry.valueSchema.fromPreparedMsgpack( + encoded.get(entry.key), + rawStringProvider.withMapValue(entry.key) + ) ); } else if (entry.omitEmpty) { map.set(entry.key, entry.valueSchema.defaultValue()); @@ -270,7 +279,8 @@ export class Uint64MapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('Uint64MapSchema data must be a Map'); @@ -281,7 +291,13 @@ export class Uint64MapSchema extends Schema { if (map.has(bigintKey)) { throw new Error(`Duplicate key: ${bigintKey}`); } - map.set(bigintKey, this.valueSchema.fromPreparedMsgpack(value)); + map.set( + bigintKey, + this.valueSchema.fromPreparedMsgpack( + value, + rawStringProvider.withMapValue(key) + ) + ); } return map; } @@ -364,7 +380,8 @@ export class StringMapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('StringMapSchema data must be a Map'); @@ -377,7 +394,13 @@ export class StringMapSchema extends Schema { if (map.has(key)) { throw new Error(`Duplicate key: ${key}`); } - map.set(key, this.valueSchema.fromPreparedMsgpack(value)); + map.set( + key, + this.valueSchema.fromPreparedMsgpack( + value, + rawStringProvider.withMapValue(key) + ) + ); } return map; } diff --git a/src/encoding/schema/optional.ts b/src/encoding/schema/optional.ts index 138928e97..96ff805c4 100644 --- a/src/encoding/schema/optional.ts +++ b/src/encoding/schema/optional.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -33,12 +38,15 @@ export class OptionalSchema extends Schema { return this.valueSchema.prepareMsgpack(data); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown { // JS undefined is encoded as msgpack nil, which may be decoded as JS null if (encoded === undefined || encoded === null) { return undefined; } - return this.valueSchema.fromPreparedMsgpack(encoded); + return this.valueSchema.fromPreparedMsgpack(encoded, rawStringProvider); } public prepareJSON(data: unknown): JSONEncodingData { diff --git a/src/encoding/schema/string.ts b/src/encoding/schema/string.ts index 2d9002209..be8be78c5 100644 --- a/src/encoding/schema/string.ts +++ b/src/encoding/schema/string.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -18,7 +23,11 @@ export class StringSchema extends Schema { throw new Error(`Invalid string: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): string { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): string { if (typeof encoded === 'string') { return encoded; } diff --git a/src/encoding/schema/uint64.ts b/src/encoding/schema/uint64.ts index c13fd5b2f..4f2cd29bb 100644 --- a/src/encoding/schema/uint64.ts +++ b/src/encoding/schema/uint64.ts @@ -1,4 +1,9 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, +} from '../encoding.js'; import { ensureUint64 } from '../../utils/utils.js'; /* eslint-disable class-methods-use-this */ @@ -18,7 +23,11 @@ export class Uint64Schema extends Schema { return ensureUint64(data); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): bigint { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): bigint { return ensureUint64(encoded); } diff --git a/src/encoding/schema/untyped.ts b/src/encoding/schema/untyped.ts index 14411ce52..1ee5ecbdb 100644 --- a/src/encoding/schema/untyped.ts +++ b/src/encoding/schema/untyped.ts @@ -1,6 +1,7 @@ import { Schema, MsgpackEncodingData, + MsgpackRawStringProvider, JSONEncodingData, msgpackEncodingDataToJSONEncodingData, jsonEncodingDataToMsgpackEncodingData, @@ -24,7 +25,9 @@ export class UntypedSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider ): MsgpackEncodingData { return encoded; } diff --git a/src/main.ts b/src/main.ts index 7143ecc75..4dfd7c1db 100644 --- a/src/main.ts +++ b/src/main.ts @@ -129,8 +129,9 @@ export { export { bytesToBigInt, bigIntToBytes } from './encoding/bigint.js'; export { base64ToBytes, - base64ToString, bytesToBase64, + bytesToString, + coerceToBytes, bytesToHex, hexToBytes, } from './encoding/binarydata.js'; diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index 05d7c7afc..36c68371b 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -1,11 +1,12 @@ /* eslint-env mocha */ import assert from 'assert'; +import { RawBinaryString } from 'algorand-msgpack'; import algosdk from '../src/index.js'; import * as utils from '../src/utils/utils.js'; import { + rawEncode as msgpackRawEncode, Schema, - MsgpackEncodingData, - JSONEncodingData, + MsgpackRawStringProvider, } from '../src/encoding/encoding.js'; import { BooleanSchema, @@ -14,6 +15,7 @@ import { AddressSchema, ByteArraySchema, FixedLengthByteArraySchema, + SpecialCaseBinaryStringSchema, ArraySchema, NamedMapSchema, NamedMapEntry, @@ -897,7 +899,9 @@ describe('encoding', () => { ['\uFFFD\uFFFD', '/v8='], // Non UTF-8 bytes should still decode to same (invalid) output ]; for (const [testCase, expectedEncoding] of testCases) { - const actualB64Decoding = algosdk.base64ToString(expectedEncoding); + const actualB64Decoding = algosdk.bytesToString( + algosdk.base64ToBytes(expectedEncoding) + ); assert.deepStrictEqual( actualB64Decoding, testCase, @@ -944,10 +948,10 @@ describe('encoding', () => { name: string; schema: Schema; values: unknown[]; - preparedMsgpackValues: MsgpackEncodingData[]; + preparedMsgpackValues: algosdk.MsgpackEncodingData[]; // The expected output from calling `fromPreparedMsgpack`. If not provided, `values` will be used. expectedValuesFromPreparedMsgpack?: unknown[]; - preparedJsonValues: JSONEncodingData[]; + preparedJsonValues: algosdk.JSONEncodingData[]; // The expected output from calling `fromPreparedJSON`. If not provided, `values` will be used. expectedValuesFromPreparedJson?: unknown[]; } @@ -967,6 +971,21 @@ describe('encoding', () => { preparedMsgpackValues: ['', 'abc'], preparedJsonValues: ['', 'abc'], }, + { + name: 'SpecialCaseBinaryStringSchema', + schema: new SpecialCaseBinaryStringSchema(), + values: [Uint8Array.from([]), Uint8Array.from([97, 98, 99])], + preparedMsgpackValues: [ + // TODO: fix cast? + new RawBinaryString( + Uint8Array.from([]) + ) as unknown as algosdk.MsgpackEncodingData, + new RawBinaryString( + Uint8Array.from([97, 98, 99]) + ) as unknown as algosdk.MsgpackEncodingData, + ], + preparedJsonValues: ['', 'abc'], + }, { name: 'Uint64Schema', schema: new Uint64Schema(), @@ -1086,6 +1105,40 @@ describe('encoding', () => { }, ], }, + { + name: 'Uint64MapSchema of SpecialCaseBinaryStringSchema', + schema: new Uint64MapSchema(new SpecialCaseBinaryStringSchema()), + values: [ + new Map(), + new Map([ + [0n, Uint8Array.from([])], + [1n, Uint8Array.from([97])], + [2n, Uint8Array.from([98])], + [BigInt('18446744073709551615'), Uint8Array.from([99])], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [0n, new RawBinaryString(Uint8Array.from([]))], + [1n, new RawBinaryString(Uint8Array.from([97]))], + [2n, new RawBinaryString(Uint8Array.from([98]))], + [ + BigInt('18446744073709551615'), + new RawBinaryString(Uint8Array.from([99])), + ], + ]), + ], + preparedJsonValues: [ + {}, + { + 0: '', + 1: 'a', + 2: 'b', + '18446744073709551615': 'c', + }, + ], + }, { name: 'StringMapSchema of BooleanSchema', schema: new StringMapSchema(new BooleanSchema()), @@ -1210,8 +1263,15 @@ describe('encoding', () => { const actualMsgpack = testcase.schema.prepareMsgpack(value); assert.deepStrictEqual(actualMsgpack, preparedMsgpackValue); - const roundtripMsgpackValue = - testcase.schema.fromPreparedMsgpack(actualMsgpack); + const msgpackBytes = msgpackRawEncode(actualMsgpack); + const rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + + const roundtripMsgpackValue = testcase.schema.fromPreparedMsgpack( + actualMsgpack, + rawStringProvider + ); const roundtripMsgpackExpectedValue = testcase.expectedValuesFromPreparedMsgpack ? testcase.expectedValuesFromPreparedMsgpack[i] @@ -1261,6 +1321,11 @@ describe('encoding', () => { emptyValue: '', nonemptyValue: 'abc', }, + { + schema: new SpecialCaseBinaryStringSchema(), + emptyValue: Uint8Array.from([]), + nonemptyValue: Uint8Array.from([97, 98, 99]), + }, { schema: new AddressSchema(), emptyValue: algosdk.Address.zeroAddress(), @@ -1396,8 +1461,14 @@ describe('encoding', () => { let prepareMsgpackResult = schema.prepareMsgpack(allEmptyValues); // All empty values should be omitted assert.deepStrictEqual(prepareMsgpackResult, new Map()); - let fromPreparedMsgpackResult = - schema.fromPreparedMsgpack(prepareMsgpackResult); + let msgpackBytes = msgpackRawEncode(prepareMsgpackResult); + let rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + let fromPreparedMsgpackResult = schema.fromPreparedMsgpack( + prepareMsgpackResult, + rawStringProvider + ); // Omitted values should be restored with their default/empty values assert.deepStrictEqual( fromPreparedMsgpackResult, @@ -1422,8 +1493,14 @@ describe('encoding', () => { assert.ok(prepareMsgpackResult instanceof Map); // All values are present assert.strictEqual(prepareMsgpackResult.size, testValues.length); - fromPreparedMsgpackResult = - schema.fromPreparedMsgpack(prepareMsgpackResult); + msgpackBytes = msgpackRawEncode(prepareMsgpackResult); + rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + fromPreparedMsgpackResult = schema.fromPreparedMsgpack( + prepareMsgpackResult, + rawStringProvider + ); // Values are restored properly assert.deepStrictEqual(fromPreparedMsgpackResult, allNonemptyValues); @@ -1766,6 +1843,9 @@ describe('encoding', () => { ); }); }); + describe('MsgpackRawStringProvider', () => { + // TODO: Add tests + }); }); describe('BlockResponse', () => { it('should decode block response correctly', () => { From 3d0f59e9b990ed59d6aa003d54debef2713c7ef7 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Tue, 23 Jul 2024 14:27:14 -0400 Subject: [PATCH 02/21] Try something different for temp msgpack branch install --- .circleci/config.yml | 10 ++++++++++ package.json | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 554e293af..db6c119b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,6 +138,16 @@ commands: << parameters.sudo >> apt -y install curl make git build-essential jq unzip - node/install: node-version: '18' + - run: + name: Install algorand-msgpack from branch (temporary) + command: | + cd .. + git clone https://github.com/algorand/msgpack-javascript.git + cd msgpack-javascript + git checkout raw-string-encoding + npm i + npm run build + npm pack - run: name: npm ci command: | diff --git a/package.json b/package.json index c991c8c9c..6a64daff0 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,6 @@ "webpack-cli": "^5.0.1" }, "scripts": { - "preinstall": "cd .. && git clone https://github.com/algorand/msgpack-javascript.git && git checkout raw-string-encoding && cd msgpack-javascript && npm i && npm run build && npm pack", "test": "tsx tests/mocha.js", "prepare": "npm run build", "prepare-browser-tests": "npm run build && mkdir -p tests/cucumber/browser/build && cp dist/browser/algosdk.min.* tests/cucumber/browser/build/ && webpack --config tests/cucumber/browser/webpack.config.js", From 0c8d5b9b8d1787f5fcb7e304dbd8fbb8da2a1b27 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Tue, 23 Jul 2024 14:29:52 -0400 Subject: [PATCH 03/21] Wrong command for msgpack build --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db6c119b6..b85d2b963 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,7 +146,7 @@ commands: cd msgpack-javascript git checkout raw-string-encoding npm i - npm run build + npm run prepare npm pack - run: name: npm ci From 9d998f2d6f26e40edd02a181a06835b75b145203 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Tue, 23 Jul 2024 17:56:56 -0400 Subject: [PATCH 04/21] More shenanigans --- tests/cucumber/docker/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index 4ef89a836..513cbdec1 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -28,6 +28,14 @@ ENV TEST_BROWSER=$TEST_BROWSER ARG CI ENV CI=$CI +# Temporary hack to install msgpack-javascript from a branch +RUN npm cd .. \ + && git clone https://github.com/algorand/msgpack-javascript.git \ + && cd msgpack-javascript \ + && git checkout raw-string-encoding \ + && npm i \ + && npm run prepare \ + && npm pack RUN npm ci RUN npm run prepare-browser-tests From 2903f154f13efef3ec8581dc929b6aff30955473 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Tue, 23 Jul 2024 18:38:28 -0400 Subject: [PATCH 05/21] Try to fix paths --- tests/cucumber/docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index 513cbdec1..1e5b803ad 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -18,9 +18,9 @@ RUN wget -q -O - https://deb.nodesource.com/setup_18.x | bash \ && echo "npm version: $(npm --version)" # Copy SDK code into the container -RUN mkdir -p $HOME/js-algorand-sdk -COPY . $HOME/js-algorand-sdk -WORKDIR $HOME/js-algorand-sdk +RUN mkdir -p /home/js-algorand-sdk +COPY . /home/js-algorand-sdk +WORKDIR /home/js-algorand-sdk ARG TEST_BROWSER ENV TEST_BROWSER=$TEST_BROWSER From 53f1686ba14f22720a9367e1b9585f2b49ca2e3c Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 10:06:33 -0400 Subject: [PATCH 06/21] Fix cd issue --- tests/cucumber/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index 1e5b803ad..d5c33144d 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -29,13 +29,13 @@ ARG CI ENV CI=$CI # Temporary hack to install msgpack-javascript from a branch -RUN npm cd .. \ +RUN bash -c "npm cd .. \ && git clone https://github.com/algorand/msgpack-javascript.git \ && cd msgpack-javascript \ && git checkout raw-string-encoding \ && npm i \ && npm run prepare \ - && npm pack + && npm pack" RUN npm ci RUN npm run prepare-browser-tests From cc5692649b703e7e8e30d52d34dd4eca7f0f19aa Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 10:30:09 -0400 Subject: [PATCH 07/21] another fix --- tests/cucumber/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index d5c33144d..1ae8af5b6 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -29,13 +29,13 @@ ARG CI ENV CI=$CI # Temporary hack to install msgpack-javascript from a branch -RUN bash -c "npm cd .. \ +RUN cd .. \ && git clone https://github.com/algorand/msgpack-javascript.git \ && cd msgpack-javascript \ && git checkout raw-string-encoding \ && npm i \ && npm run prepare \ - && npm pack" + && npm pack RUN npm ci RUN npm run prepare-browser-tests From 593cf2e233477711aa3e36f3cd3ba4e7a39d7b1e Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 10:59:23 -0400 Subject: [PATCH 08/21] No git? No problem --- tests/cucumber/docker/Dockerfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index 1ae8af5b6..c28eac63e 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -22,20 +22,16 @@ RUN mkdir -p /home/js-algorand-sdk COPY . /home/js-algorand-sdk WORKDIR /home/js-algorand-sdk +# Temporary hack to copy in msgpack-javascript from a branch +RUN mkdir -p /home/msgpack-javascript +COPY ../msgpack-javascript/algorand-msgpack-1.0.1.tgz /home/msgpack-javascript/algorand-msgpack-1.0.1.tgz + ARG TEST_BROWSER ENV TEST_BROWSER=$TEST_BROWSER ARG CI ENV CI=$CI -# Temporary hack to install msgpack-javascript from a branch -RUN cd .. \ - && git clone https://github.com/algorand/msgpack-javascript.git \ - && cd msgpack-javascript \ - && git checkout raw-string-encoding \ - && npm i \ - && npm run prepare \ - && npm pack RUN npm ci RUN npm run prepare-browser-tests From 96994378512b4c059d0e88e3ab280cd01ed1b14b Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 11:25:42 -0400 Subject: [PATCH 09/21] Move file into same folder --- .circleci/config.yml | 1 + tests/cucumber/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b85d2b963..0ab671b37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,6 +148,7 @@ commands: npm i npm run prepare npm pack + cp algorand-msgpack-1.0.1.tgz ../js-algorand-sdk/algorand-msgpack-1.0.1.tgz - run: name: npm ci command: | diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index c28eac63e..43f7d2e65 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -24,7 +24,7 @@ WORKDIR /home/js-algorand-sdk # Temporary hack to copy in msgpack-javascript from a branch RUN mkdir -p /home/msgpack-javascript -COPY ../msgpack-javascript/algorand-msgpack-1.0.1.tgz /home/msgpack-javascript/algorand-msgpack-1.0.1.tgz +COPY algorand-msgpack-1.0.1.tgz /home/msgpack-javascript/algorand-msgpack-1.0.1.tgz ARG TEST_BROWSER ENV TEST_BROWSER=$TEST_BROWSER From 615b6392b34912f7b8cd972aa7484b7a047a0898 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 11:37:48 -0400 Subject: [PATCH 10/21] something else --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ab671b37..8d2343173 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,6 +141,7 @@ commands: - run: name: Install algorand-msgpack from branch (temporary) command: | + set -e cd .. git clone https://github.com/algorand/msgpack-javascript.git cd msgpack-javascript @@ -148,10 +149,10 @@ commands: npm i npm run prepare npm pack - cp algorand-msgpack-1.0.1.tgz ../js-algorand-sdk/algorand-msgpack-1.0.1.tgz - run: name: npm ci command: | set -e npm ci + cp ../msgpack-javascript/algorand-msgpack-1.0.1.tgz algorand-msgpack-1.0.1.tgz if [ "<< parameters.browser >>" == "chrome" ]; then npm install chromedriver@latest; fi From 5eb20deb1917637ec4368fcbff1ed0eff4ad17f9 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 13:02:21 -0400 Subject: [PATCH 11/21] Remove old usage of base64ToString --- examples/app.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/app.ts b/examples/app.ts index 775eed265..0c93150a3 100644 --- a/examples/app.ts +++ b/examples/app.ts @@ -132,11 +132,10 @@ async function main() { // example: APP_READ_STATE const appInfo = await algodClient.getApplicationByID(appId).do(); - const globalState = appInfo.params.globalState[0]; - console.log(`Raw global state - ${algosdk.stringifyJSON(globalState)}`); + const globalState = appInfo.params.globalState![0]; + console.log(`Raw global state - ${algosdk.encodeJSON(globalState)}`); - // decode b64 string key with Buffer - const globalKey = algosdk.base64ToString(globalState.key); + const globalKey = globalState.key; // show global value const globalValue = globalState.value.bytes; @@ -146,11 +145,10 @@ async function main() { .accountApplicationInformation(caller.addr, appId) .do(); - const localState = accountAppInfo.appLocalState.keyValue[0]; + const localState = accountAppInfo.appLocalState!.keyValue![0]; console.log(`Raw local state - ${algosdk.stringifyJSON(localState)}`); - // decode b64 string key with Buffer - const localKey = algosdk.base64ToString(localState.key); + const localKey = localState.key; // get uint value directly const localValue = localState.value.uint; From de266e89815cec31d368b5730760447d18c07d04 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 24 Jul 2024 15:26:46 -0400 Subject: [PATCH 12/21] Implement SpecialCaseBinaryStringMapSchema --- src/encoding/encoding.ts | 30 +++++++---- src/encoding/schema/index.ts | 1 + src/encoding/schema/map.ts | 99 ++++++++++++++++++++++++++++++++++++ tests/2.Encoding.ts | 87 +++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 10 deletions(-) diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index cfb1b0b0e..33a212454 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -179,7 +179,7 @@ enum MsgpackObjectPathSegmentKind { interface MsgpackObjectPathSegment { kind: MsgpackObjectPathSegmentKind; - key: string | number | bigint | Uint8Array; + key: string | number | bigint | Uint8Array | RawBinaryString; } export class MsgpackRawStringProvider { @@ -214,7 +214,7 @@ export class MsgpackRawStringProvider { } public withMapValue( - key: string | number | bigint | Uint8Array + key: string | number | bigint | Uint8Array | RawBinaryString ): MsgpackRawStringProvider { return new MsgpackRawStringProvider({ parent: this, @@ -246,25 +246,28 @@ export class MsgpackRawStringProvider { ); } - public getRawStringKeysAtCurrentLocation(): Uint8Array[] { + public getRawStringKeysAndValuesAtCurrentLocation(): Map< + Uint8Array, + MsgpackEncodingData + > { const resolved = this.resolve(); if (!(resolved instanceof Map)) { throw new Error( `Invalid type. Expected Map, got ${resolved} (${typeof resolved})` ); } - const keys: Uint8Array[] = []; - for (const key of resolved.keys()) { + const keysAndValues = new Map(); + for (const [key, value] of resolved) { if (key instanceof RawBinaryString) { // Decoded rawBinaryValue will always be a Uint8Array - keys.push(key.rawBinaryValue as Uint8Array); + keysAndValues.set(key.rawBinaryValue as Uint8Array, value); } else { throw new Error( `Invalid type for map key. Expected RawBinaryString, got (${typeof key}) ${key}` ); } } - return keys; + return keysAndValues; } private resolve(): MsgpackEncodingData { @@ -295,10 +298,17 @@ export class MsgpackRawStringProvider { // so we must check all the values for value-equality. if ( typeof this.segment.key === 'string' || - this.segment.key instanceof Uint8Array + this.segment.key instanceof Uint8Array || + this.segment.key instanceof RawBinaryString ) { - const targetBytes = coerceToBytes(this.segment.key); - const targetIsRawString = typeof this.segment.key === 'string'; + const targetBytes = + this.segment.key instanceof RawBinaryString + ? // Decoded rawBinaryValue will always be a Uint8Array + (this.segment.key.rawBinaryValue as Uint8Array) + : coerceToBytes(this.segment.key); + const targetIsRawString = + typeof this.segment.key === 'string' || + this.segment.key instanceof RawBinaryString; for (const [key, value] of parentResolved) { let potentialKeyBytes: Uint8Array | undefined; if (targetIsRawString) { diff --git a/src/encoding/schema/index.ts b/src/encoding/schema/index.ts index 2ae03c2c5..208799132 100644 --- a/src/encoding/schema/index.ts +++ b/src/encoding/schema/index.ts @@ -16,6 +16,7 @@ export { convertMap, Uint64MapSchema, StringMapSchema, + SpecialCaseBinaryStringMapSchema, } from './map.js'; export { OptionalSchema } from './optional.js'; diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 7ce968e1f..b535f2527 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -1,3 +1,4 @@ +import { RawBinaryString } from 'algorand-msgpack'; import { Schema, MsgpackEncodingData, @@ -5,6 +6,7 @@ import { JSONEncodingData, } from '../encoding.js'; import { ensureUint64 } from '../../utils/utils.js'; +import { bytesToString, coerceToBytes } from '../binarydata.js'; /* eslint-disable class-methods-use-this */ @@ -447,3 +449,100 @@ export class StringMapSchema extends Schema { return map; } } + +/** + * Schema for a map with a variable number of binary string keys. + * + * See SpecialCaseBinaryStringSchema for more information about the key type. + */ +export class SpecialCaseBinaryStringMapSchema extends Schema { + constructor(public readonly valueSchema: Schema) { + super(); + } + + public defaultValue(): Map { + return new Map(); + } + + public isDefaultValue(data: unknown): boolean { + return data instanceof Map && data.size === 0; + } + + public prepareMsgpack(data: unknown): MsgpackEncodingData { + if (!(data instanceof Map)) { + throw new Error( + `SpecialCaseBinaryStringMapSchema data must be a Map. Got (${typeof data}) ${data}` + ); + } + const prepared = new Map(); + for (const [key, value] of data) { + if (!(key instanceof Uint8Array)) { + throw new Error(`Invalid key: ${key} (${typeof key})`); + } + prepared.set( + new RawBinaryString(key), + this.valueSchema.prepareMsgpack(value) + ); + } + // TODO: fix cast? + return prepared as unknown as Map; + } + + public fromPreparedMsgpack( + _encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): Map { + const map = new Map(); + const keysAndValues = + rawStringProvider.getRawStringKeysAndValuesAtCurrentLocation(); + for (const [key, value] of keysAndValues) { + map.set( + key, + this.valueSchema.fromPreparedMsgpack( + value, + rawStringProvider.withMapValue(new RawBinaryString(key)) + ) + ); + } + return map; + } + + public prepareJSON(data: unknown): JSONEncodingData { + if (!(data instanceof Map)) { + throw new Error( + `SpecialCaseBinaryStringMapSchema data must be a Map. Got (${typeof data}) ${data}` + ); + } + const prepared = new Map(); + for (const [key, value] of data) { + if (!(key instanceof Uint8Array)) { + throw new Error(`Invalid key: ${key}`); + } + // WARNING: not safe for all binary data + prepared.set(bytesToString(key), this.valueSchema.prepareJSON(value)); + } + // Convert map to object + const obj: { [key: string]: JSONEncodingData } = {}; + for (const [key, value] of prepared) { + obj[key] = value; + } + return obj; + } + + public fromPreparedJSON(encoded: JSONEncodingData): Map { + if ( + encoded == null || + typeof encoded !== 'object' || + Array.isArray(encoded) + ) { + throw new Error( + 'SpecialCaseBinaryStringMapSchema data must be an object' + ); + } + const map = new Map(); + for (const [key, value] of Object.entries(encoded)) { + map.set(coerceToBytes(key), this.valueSchema.fromPreparedJSON(value)); + } + return map; + } +} diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index 36c68371b..23a9d9049 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -21,6 +21,7 @@ import { NamedMapEntry, Uint64MapSchema, StringMapSchema, + SpecialCaseBinaryStringMapSchema, UntypedSchema, OptionalSchema, allOmitEmpty, @@ -1170,6 +1171,82 @@ describe('encoding', () => { }, ], }, + { + name: 'SpecialCaseBinaryStringMapSchema of BooleanSchema', + schema: new SpecialCaseBinaryStringMapSchema(new BooleanSchema()), + values: [ + new Map(), + new Map([ + [Uint8Array.from([97]), true], + [Uint8Array.from([98]), false], + [Uint8Array.from([99]), true], + [Uint8Array.from([]), true], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [new RawBinaryString(Uint8Array.from([97])), true], + [new RawBinaryString(Uint8Array.from([98])), false], + [new RawBinaryString(Uint8Array.from([99])), true], + [new RawBinaryString(Uint8Array.from([])), true], + ]), + ], + preparedJsonValues: [ + {}, + { + a: true, + b: false, + c: true, + '': true, + }, + ], + }, + { + name: 'SpecialCaseBinaryStringMapSchema of SpecialCaseBinaryStringSchema', + schema: new SpecialCaseBinaryStringMapSchema( + new SpecialCaseBinaryStringSchema() + ), + values: [ + new Map(), + new Map([ + [Uint8Array.from([97]), Uint8Array.from([120])], + [Uint8Array.from([98]), Uint8Array.from([121])], + [Uint8Array.from([99]), Uint8Array.from([122])], + [Uint8Array.from([]), Uint8Array.from([])], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [ + new RawBinaryString(Uint8Array.from([97])), + new RawBinaryString(Uint8Array.from([120])), + ], + [ + new RawBinaryString(Uint8Array.from([98])), + new RawBinaryString(Uint8Array.from([121])), + ], + [ + new RawBinaryString(Uint8Array.from([99])), + new RawBinaryString(Uint8Array.from([122])), + ], + [ + new RawBinaryString(Uint8Array.from([])), + new RawBinaryString(Uint8Array.from([])), + ], + ]), + ], + preparedJsonValues: [ + {}, + { + a: 'x', + b: 'y', + c: 'z', + '': '', + }, + ], + }, ]; const primitiveTestcases = testcases.slice(); @@ -1409,6 +1486,16 @@ describe('encoding', () => { ['', true], ]), }, + { + schema: new SpecialCaseBinaryStringMapSchema(new BooleanSchema()), + emptyValue: new Map(), + nonemptyValue: new Map([ + [Uint8Array.from([97]), true], + [Uint8Array.from([98]), false], + [Uint8Array.from([99]), true], + [Uint8Array.from([]), true], + ]), + }, ]; for (const testValue of testValues.slice()) { From df1a9744c86a3685723a29b860a1b77e8f4e1018 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Thu, 25 Jul 2024 15:21:51 -0400 Subject: [PATCH 13/21] Use raw string schemas in Block decoding --- src/encoding/encoding.ts | 121 ++++++++++++++++++++++++++++++------- src/encoding/schema/map.ts | 31 +++++++++- src/group.ts | 4 +- src/main.ts | 24 ++------ src/transaction.ts | 2 +- src/types/block.ts | 45 ++++++++------ tests/2.Encoding.ts | 51 ++++++++-------- tsdoc.json | 4 +- 8 files changed, 190 insertions(+), 92 deletions(-) diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 33a212454..7d3dbfad0 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -44,56 +44,115 @@ function containsEmpty(obj: Record) { } /** - * rawEncode encodes objects using msgpack, regardless of whether there are + * msgpackRawEncode encodes objects using msgpack, regardless of whether there are * empty or 0 value fields. * @param obj - a dictionary to be encoded. May or may not contain empty or 0 values. * @returns msgpack representation of the object */ -export function rawEncode(obj: unknown) { +export function msgpackRawEncode(obj: unknown) { // enable the canonical option const options: EncoderOptions = { sortKeys: true }; return msgpackEncode(obj, options); } /** - * encode encodes objects using msgpack - * @param obj - a dictionary to be encoded. Must not contain empty or 0 values. - * @returns msgpack representation of the object + * encodeObj takes a javascript object and returns its msgpack encoding + * Note that the encoding sorts the fields alphabetically + * @param o - js object to be encoded. Must not contain empty or 0 values. + * @returns Uint8Array binary representation * @throws Error containing ERROR_CONTAINS_EMPTY_STRING if the object contains empty or zero values + * + * @deprecated Use {@link msgpackRawEncode} instead. Note that function does not + * check for empty values like this one does. */ -export function encode(obj: Record) { +export function encodeObj(obj: Record) { // Check for empty values const emptyCheck = containsEmpty(obj); if (emptyCheck.containsEmpty) { throw new Error(ERROR_CONTAINS_EMPTY_STRING + emptyCheck.firstEmptyKey); } + return msgpackRawEncode(obj); +} - // enable the canonical option - return rawEncode(obj); +function intDecodingToIntMode(intDecoding: IntDecoding): IntMode { + switch (intDecoding) { + case IntDecoding.UNSAFE: + return IntMode.UNSAFE_NUMBER; + case IntDecoding.SAFE: + return IntMode.SAFE_NUMBER; + case IntDecoding.MIXED: + return IntMode.MIXED; + case IntDecoding.BIGINT: + return IntMode.BIGINT; + default: + throw new Error(`Invalid intDecoding: ${intDecoding}`); + } } -export function decode(buffer: ArrayLike) { - // TODO: make IntMode an argument - const options: DecoderOptions = { intMode: IntMode.MIXED }; - return msgpackDecode(buffer, options); +/** + * Decodes msgpack bytes into a plain JavaScript object. + * @param buffer - The msgpack bytes to decode + * @param options - Options for decoding, including int decoding mode. See {@link IntDecoding} for more information. + * @returns The decoded object + */ +export function msgpackRawDecode( + buffer: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, + }; + return msgpackDecode(buffer, decoderOptions); } -export function decodeAsMap(encoded: ArrayLike) { - // TODO: make IntMode an argument - const options: DecoderOptions = { intMode: IntMode.MIXED, useMap: true }; - return msgpackDecode(encoded, options); +/** + * decodeObj takes a Uint8Array and returns its javascript obj + * @param o - Uint8Array to decode + * @returns object + * + * @deprecated Use {@link msgpackRawDecode} instead. Note that this function uses `IntDecoding.MIXED` + * while `msgpackRawDecode` defaults to `IntDecoding.BIGINT` for int decoding, though it is + * configurable. + */ +export function decodeObj(o: ArrayLike) { + return msgpackRawDecode(o, { intDecoding: IntDecoding.MIXED }); } -function decodeAsMapWithRawStrings(encoded: ArrayLike) { - // TODO: make IntMode an argument - const options: DecoderOptions = { - intMode: IntMode.BIGINT, +/** + * Decodes msgpack bytes into a Map object. This supports decoding non-string map keys. + * @param encoded - The msgpack bytes to decode + * @param options - Options for decoding, including int decoding mode. See {@link IntDecoding} for more information. + * @returns The decoded Map object + */ +export function decodeAsMap( + encoded: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, + useMap: true, + }; + return msgpackDecode(encoded, decoderOptions); +} + +function decodeAsMapWithRawStrings( + encoded: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, useMap: true, rawBinaryStringKeys: true, rawBinaryStringValues: true, useRawBinaryStringClass: true, }; - return msgpackDecode(encoded, options); + return msgpackDecode(encoded, decoderOptions); } export type MsgpackEncodingData = @@ -342,6 +401,20 @@ export class MsgpackRawStringProvider { } throw new Error(`Invalid segment kind: ${this.segment.kind}`); } + + public getPathString(): string { + const parentPathString = this.parent ? this.parent.getPathString() : 'root'; + if (!this.segment) { + return parentPathString; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.MAP_VALUE) { + return `${parentPathString} -> map key "${this.segment.key}" (${typeof this.segment.key})`; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.ARRAY_ELEMENT) { + return `${parentPathString} -> array index ${this.segment.key} (${typeof this.segment.key})`; + } + return `${parentPathString} -> unknown segment kind ${this.segment.kind}`; + } } /** @@ -450,7 +523,9 @@ export function decodeMsgpack( * @returns A msgpack byte array encoding of the object */ export function encodeMsgpack(e: Encodable): Uint8Array { - return rawEncode(e.getEncodingSchema().prepareMsgpack(e.toEncodingData())); + return msgpackRawEncode( + e.getEncodingSchema().prepareMsgpack(e.toEncodingData()) + ); } /** @@ -464,7 +539,7 @@ export function decodeJSON( c: EncodableClass ): T { const decoded: JSONEncodingData = parseJSON(encoded, { - intDecoding: IntDecoding.MIXED, + intDecoding: IntDecoding.BIGINT, }); return c.fromEncodingData( c.encodingSchema.fromPreparedJSON(decoded) as JSONEncodingData diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index b535f2527..83e788a0b 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -450,6 +450,35 @@ export class StringMapSchema extends Schema { } } +function removeRawStringsFromMsgpackValues( + value: MsgpackEncodingData +): MsgpackEncodingData { + if (value instanceof RawBinaryString) { + return bytesToString(value.rawBinaryValue as Uint8Array); + } + if (value instanceof Map) { + const newMap = new Map< + string | number | bigint | Uint8Array, + MsgpackEncodingData + >(); + for (const [key, val] of value) { + newMap.set( + removeRawStringsFromMsgpackValues(key) as + | string + | number + | bigint + | Uint8Array, + removeRawStringsFromMsgpackValues(val) + ); + } + return newMap; + } + if (Array.isArray(value)) { + return value.map(removeRawStringsFromMsgpackValues); + } + return value; +} + /** * Schema for a map with a variable number of binary string keys. * @@ -499,7 +528,7 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { map.set( key, this.valueSchema.fromPreparedMsgpack( - value, + removeRawStringsFromMsgpackValues(value), rawStringProvider.withMapValue(new RawBinaryString(key)) ) ); diff --git a/src/group.ts b/src/group.ts index 16c8d50c0..f870f518d 100644 --- a/src/group.ts +++ b/src/group.ts @@ -1,6 +1,6 @@ import { Transaction } from './transaction.js'; import * as nacl from './nacl/naclWrappers.js'; -import * as encoding from './encoding/encoding.js'; +import { encodeObj } from './encoding/encoding.js'; import * as utils from './utils/utils.js'; const ALGORAND_MAX_TX_GROUP_SIZE = 16; @@ -12,7 +12,7 @@ function txGroupPreimage(txnHashes: Uint8Array[]): Uint8Array { `${txnHashes.length} transactions grouped together but max group size is ${ALGORAND_MAX_TX_GROUP_SIZE}` ); } - const bytes = encoding.encode({ + const bytes = encodeObj({ txlist: txnHashes, }); return utils.concatArrays(TX_GROUP_TAG, bytes); diff --git a/src/main.ts b/src/main.ts index 4dfd7c1db..7488baba9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,6 +1,5 @@ import * as nacl from './nacl/naclWrappers.js'; import { Address } from './encoding/address.js'; -import * as encoding from './encoding/encoding.js'; import { Transaction } from './transaction.js'; import * as convert from './convert.js'; import * as utils from './utils/utils.js'; @@ -64,25 +63,6 @@ export function verifyBytes( return nacl.verify(toBeVerified, signature, addrObj.publicKey); } -/** - * encodeObj takes a javascript object and returns its msgpack encoding - * Note that the encoding sorts the fields alphabetically - * @param o - js obj - * @returns Uint8Array binary representation - */ -export function encodeObj(o: Record) { - return new Uint8Array(encoding.encode(o)); -} - -/** - * decodeObj takes a Uint8Array and returns its javascript obj - * @param o - Uint8Array to decode - * @returns object - */ -export function decodeObj(o: ArrayLike) { - return encoding.decode(o); -} - export const ERROR_MULTISIG_BAD_SENDER = new Error( MULTISIG_BAD_SENDER_ERROR_MSG ); @@ -113,6 +93,10 @@ export { JSONEncodingData, Encodable, EncodableClass, + encodeObj, + decodeObj, + msgpackRawEncode, + msgpackRawDecode, encodeMsgpack, decodeMsgpack, encodeJSON, diff --git a/src/transaction.ts b/src/transaction.ts index 06c3a6cc2..b27b63617 100644 --- a/src/transaction.ts +++ b/src/transaction.ts @@ -1088,7 +1088,7 @@ export class Transaction implements encoding.Encodable { ]) ); - return encoding.rawEncode(stxnSchema.prepareMsgpack(sTxn)); + return encoding.msgpackRawEncode(stxnSchema.prepareMsgpack(sTxn)); } rawTxID(): Uint8Array { diff --git a/src/types/block.ts b/src/types/block.ts index 08256be3b..d90e207fd 100644 --- a/src/types/block.ts +++ b/src/types/block.ts @@ -2,7 +2,8 @@ import { Encodable, Schema } from '../encoding/encoding.js'; import { NamedMapSchema, Uint64MapSchema, - StringMapSchema, + SpecialCaseBinaryStringMapSchema, + SpecialCaseBinaryStringSchema, ArraySchema, StringSchema, BooleanSchema, @@ -768,7 +769,7 @@ export class ValueDelta implements Encodable { }, { key: 'bs', // bytes - valueSchema: new StringSchema(), + valueSchema: new SpecialCaseBinaryStringSchema(), }, { key: 'ui', // uint @@ -778,10 +779,14 @@ export class ValueDelta implements Encodable { ); public action: number; - public bytes: string; + public bytes: Uint8Array; public uint: bigint; - public constructor(params: { action: number; bytes: string; uint: bigint }) { + public constructor(params: { + action: number; + bytes: Uint8Array; + uint: bigint; + }) { this.action = params.action; this.bytes = params.bytes; this.uint = params.uint; @@ -825,14 +830,14 @@ export class EvalDelta implements Encodable { { key: 'gd', // globalDelta valueSchema: new OptionalSchema( - new StringMapSchema(ValueDelta.encodingSchema) + new SpecialCaseBinaryStringMapSchema(ValueDelta.encodingSchema) ), }, { key: 'ld', // localDeltas valueSchema: new OptionalSchema( new Uint64MapSchema( - new StringMapSchema(ValueDelta.encodingSchema) + new SpecialCaseBinaryStringMapSchema(ValueDelta.encodingSchema) ) ), }, @@ -845,7 +850,7 @@ export class EvalDelta implements Encodable { { key: 'lg', // logs valueSchema: new OptionalSchema( - new ArraySchema(new StringSchema()) + new ArraySchema(new SpecialCaseBinaryStringSchema()) ), }, { @@ -861,13 +866,13 @@ export class EvalDelta implements Encodable { return this.encodingSchemaValue; } - public globalDelta: Map; + public globalDelta: Map; /** * When decoding EvalDeltas, the integer key represents an offset into * [txn.Sender, txn.Accounts[0], txn.Accounts[1], ...] */ - public localDeltas: Map>; + public localDeltas: Map>; /** * If a program modifies the local of an account that is not the Sender, or @@ -876,22 +881,22 @@ export class EvalDelta implements Encodable { */ public sharedAccts: Address[]; - public logs: string[]; + public logs: Uint8Array[]; // eslint-disable-next-line no-use-before-define public innerTxns: SignedTxnWithAD[]; public constructor(params: { - globalDelta?: Map; - localDeltas?: Map>; + globalDelta?: Map; + localDeltas?: Map>; sharedAccts?: Address[]; - logs?: string[]; + logs?: Uint8Array[]; // eslint-disable-next-line no-use-before-define innerTxns?: SignedTxnWithAD[]; }) { - this.globalDelta = params.globalDelta ?? new Map(); + this.globalDelta = params.globalDelta ?? new Map(); this.localDeltas = - params.localDeltas ?? new Map>(); + params.localDeltas ?? new Map>(); this.sharedAccts = params.sharedAccts ?? []; this.logs = params.logs ?? []; this.innerTxns = params.innerTxns ?? []; @@ -930,14 +935,14 @@ export class EvalDelta implements Encodable { } return new EvalDelta({ globalDelta: data.get('gd') - ? convertMap(data.get('gd') as Map, (key, value) => [ - key, - ValueDelta.fromEncodingData(value), - ]) + ? convertMap( + data.get('gd') as Map, + (key, value) => [key, ValueDelta.fromEncodingData(value)] + ) : undefined, localDeltas: data.get('ld') ? convertMap( - data.get('ld') as Map>, + data.get('ld') as Map>, (key, value) => [ Number(key), convertMap(value, (k, v) => [k, ValueDelta.fromEncodingData(v)]), diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index 23a9d9049..e0b350bff 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -3,11 +3,7 @@ import assert from 'assert'; import { RawBinaryString } from 'algorand-msgpack'; import algosdk from '../src/index.js'; import * as utils from '../src/utils/utils.js'; -import { - rawEncode as msgpackRawEncode, - Schema, - MsgpackRawStringProvider, -} from '../src/encoding/encoding.js'; +import { Schema, MsgpackRawStringProvider } from '../src/encoding/encoding.js'; import { BooleanSchema, StringSchema, @@ -1340,7 +1336,7 @@ describe('encoding', () => { const actualMsgpack = testcase.schema.prepareMsgpack(value); assert.deepStrictEqual(actualMsgpack, preparedMsgpackValue); - const msgpackBytes = msgpackRawEncode(actualMsgpack); + const msgpackBytes = algosdk.msgpackRawEncode(actualMsgpack); const rawStringProvider = new MsgpackRawStringProvider({ baseObjectBytes: msgpackBytes, }); @@ -1548,7 +1544,7 @@ describe('encoding', () => { let prepareMsgpackResult = schema.prepareMsgpack(allEmptyValues); // All empty values should be omitted assert.deepStrictEqual(prepareMsgpackResult, new Map()); - let msgpackBytes = msgpackRawEncode(prepareMsgpackResult); + let msgpackBytes = algosdk.msgpackRawEncode(prepareMsgpackResult); let rawStringProvider = new MsgpackRawStringProvider({ baseObjectBytes: msgpackBytes, }); @@ -1580,7 +1576,7 @@ describe('encoding', () => { assert.ok(prepareMsgpackResult instanceof Map); // All values are present assert.strictEqual(prepareMsgpackResult.size, testValues.length); - msgpackBytes = msgpackRawEncode(prepareMsgpackResult); + msgpackBytes = algosdk.msgpackRawEncode(prepareMsgpackResult); rawStringProvider = new MsgpackRawStringProvider({ baseObjectBytes: msgpackBytes, }); @@ -2090,8 +2086,8 @@ describe('encoding', () => { ], ]), ], - ['rnd', 94], - ['step', 2], + ['rnd', BigInt(94)], + ['step', BigInt(2)], [ 'vote', [ @@ -2181,47 +2177,47 @@ describe('encoding', () => { configAsset: BigInt(7777), applicationID: BigInt(8888), evalDelta: new algosdk.EvalDelta({ - globalDelta: new Map([ + globalDelta: new Map([ [ - 'globalKey1', + algosdk.coerceToBytes('globalKey1'), new algosdk.ValueDelta({ action: 1, uint: BigInt(0), - bytes: 'abc', + bytes: algosdk.coerceToBytes('abc'), }), ], [ - 'globalKey2', + algosdk.coerceToBytes('globalKey2'), new algosdk.ValueDelta({ action: 2, uint: BigInt(50), - bytes: '', + bytes: new Uint8Array(), }), ], ]), - localDeltas: new Map>([ + localDeltas: new Map>([ [ 0, - new Map([ + new Map([ [ - 'localKey1', + algosdk.coerceToBytes('localKey1'), new algosdk.ValueDelta({ action: 1, uint: BigInt(0), - bytes: 'def', + bytes: algosdk.coerceToBytes('def'), }), ], ]), ], [ 2, - new Map([ + new Map([ [ - 'localKey2', + algosdk.coerceToBytes('localKey2'), new algosdk.ValueDelta({ action: 2, uint: BigInt(51), - bytes: '', + bytes: new Uint8Array(), }), ], ]), @@ -2233,7 +2229,7 @@ describe('encoding', () => { ), algosdk.Address.zeroAddress(), ], - logs: ['log1', 'log2'], + logs: [algosdk.coerceToBytes('log1'), algosdk.coerceToBytes('log2')], innerTxns: [ new algosdk.SignedTxnWithAD({ signedTxn: new algosdk.SignedTransaction({ @@ -2259,13 +2255,20 @@ describe('encoding', () => { }), applyData: new algosdk.ApplyData({ evalDelta: new algosdk.EvalDelta({ - logs: ['log3', 'log4'], + logs: [ + algosdk.coerceToBytes('log3'), + algosdk.coerceToBytes('log4'), + ], }), }), }), ], }), }); + assert.deepStrictEqual( + algosdk.encodeJSON(applyData, 2), + algosdk.encodeJSON(expectedApplyData, 2) + ); assert.deepStrictEqual(applyData, expectedApplyData); const reencoded = algosdk.encodeMsgpack(applyData); assert.deepStrictEqual(reencoded, encodedApplyData); diff --git a/tsdoc.json b/tsdoc.json index a21a07602..46e8bac58 100644 --- a/tsdoc.json +++ b/tsdoc.json @@ -16,6 +16,8 @@ "@throws": true, "@returns": true, "@param": true, - "@category": true + "@category": true, + "@deprecated": true, + "@link": true } } From a38f991e80dea5d448759b9eefb7ec279718b126 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Fri, 26 Jul 2024 12:05:41 -0400 Subject: [PATCH 14/21] Additional test coverage --- src/encoding/encoding.ts | 4 +- tests/2.Encoding.ts | 275 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 272 insertions(+), 7 deletions(-) diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 7d3dbfad0..f47f08e03 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -247,7 +247,7 @@ export class MsgpackRawStringProvider { private readonly baseObjectBytes?: ArrayLike; - private readonly segment?; + private readonly segment?: MsgpackObjectPathSegment; private resolvedCache: MsgpackEncodingData = null; private resolvedCachePresent = false; @@ -322,7 +322,7 @@ export class MsgpackRawStringProvider { keysAndValues.set(key.rawBinaryValue as Uint8Array, value); } else { throw new Error( - `Invalid type for map key. Expected RawBinaryString, got (${typeof key}) ${key}` + `Invalid type for map key. Expected RawBinaryString, got ${key} (${typeof key})` ); } } diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index e0b350bff..3c8d12eaf 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -1927,7 +1927,160 @@ describe('encoding', () => { }); }); describe('MsgpackRawStringProvider', () => { - // TODO: Add tests + it('correctly records paths and provides raw strings', () => { + const baseObject = new Map([ + ['a', new Map([['a1', 'abc']])], + ['b', [new Map(), new Map([[BigInt(17), 'def']])]], + ]); + const baseProvider = new MsgpackRawStringProvider({ + baseObjectBytes: algosdk.msgpackRawEncode(baseObject), + }); + assert.strictEqual(baseProvider.getPathString(), 'root'); + assert.throws( + () => baseProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + baseProvider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map([ + [ + Uint8Array.from([97]), + new Map([ + [ + new RawBinaryString(Uint8Array.from([97, 49])), + new RawBinaryString(Uint8Array.from([97, 98, 99])), + ], + ]), + ], + [ + Uint8Array.from([98]), + [ + new Map(), + new Map([ + [ + BigInt(17), + new RawBinaryString(Uint8Array.from([100, 101, 102])), + ], + ]), + ], + ], + ]) + ); + + // Test with both string and raw string form + for (const firstKey of [ + 'a', + new RawBinaryString(Uint8Array.from([97])), + ]) { + const firstValueProvider = baseProvider.withMapValue(firstKey); + assert.strictEqual( + firstValueProvider.getPathString(), + `root -> map key "${firstKey}" (${typeof firstKey})` + ); + assert.throws( + () => firstValueProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + firstValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map([ + [ + Uint8Array.from([97, 49]), + new RawBinaryString(Uint8Array.from([97, 98, 99])), + ], + ]) + ); + + // Test with both string and raw string form + for (const firstFirstKey of [ + 'a1', + new RawBinaryString(Uint8Array.from([97, 49])), + ]) { + const firstFirstValueProvider = + firstValueProvider.withMapValue(firstFirstKey); + assert.strictEqual( + firstFirstValueProvider.getPathString(), + `root -> map key "${firstKey}" (${typeof firstKey}) -> map key "${firstFirstKey}" (${typeof firstFirstKey})` + ); + assert.deepStrictEqual( + firstFirstValueProvider.getRawStringAtCurrentLocation(), + Uint8Array.from([97, 98, 99]) + ); + assert.throws( + () => + firstFirstValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + } + } + + // Test with both string and raw string form + for (const secondKey of [ + 'b', + new RawBinaryString(Uint8Array.from([98])), + ]) { + const secondValueProvider = baseProvider.withMapValue(secondKey); + assert.strictEqual( + secondValueProvider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey})` + ); + assert.throws( + () => secondValueProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.throws( + () => + secondValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + + const secondIndex0Provider = secondValueProvider.withArrayElement(0); + assert.strictEqual( + secondIndex0Provider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 0 (number)` + ); + assert.throws( + () => secondIndex0Provider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + secondIndex0Provider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map() + ); + + const secondIndex1Provider = secondValueProvider.withArrayElement(1); + assert.strictEqual( + secondIndex1Provider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 1 (number)` + ); + assert.throws( + () => secondIndex1Provider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.throws( + () => + secondIndex1Provider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type for map key\. Expected RawBinaryString, got 17 \(bigint\)/ + ); + + const secondIndex1FirstProvider = secondIndex1Provider.withMapValue( + BigInt(17) + ); + assert.strictEqual( + secondIndex1FirstProvider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 1 (number) -> map key "17" (bigint)` + ); + assert.deepStrictEqual( + secondIndex1FirstProvider.getRawStringAtCurrentLocation(), + Uint8Array.from([100, 101, 102]) + ); + assert.throws( + () => + secondIndex1FirstProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + } + }); }); }); describe('BlockResponse', () => { @@ -2265,13 +2418,125 @@ describe('encoding', () => { ], }), }); - assert.deepStrictEqual( - algosdk.encodeJSON(applyData, 2), - algosdk.encodeJSON(expectedApplyData, 2) - ); assert.deepStrictEqual(applyData, expectedApplyData); const reencoded = algosdk.encodeMsgpack(applyData); assert.deepStrictEqual(reencoded, encodedApplyData); }); + it('should decode EvalDelta with invalid UTF-8 strings correctly', () => { + const encodedEvalDelta = algosdk.base64ToBytes( + 'haJnZIKkZzH+/4KiYXQBomJzpHYx/v+kZzL+/4KiYXQConVpMqNpdHiRgqJkdIGibGeSpmxvZzP+/6b+/2xvZzSjdHhuhqRhcGlkzR5ho2ZlZc0D6KJmdlyibHbNBESjc25kxCB/k/5DqCp+P1WW/80hkucvP2neluTb4OL7yjQQnAxn6aR0eXBlpGFwcGyibGSCAIGkbDH+/4KiYXQBomJzpHYy/v8CgaRsMv7/gqJhdAKidWkzomxnkqZsb2cx/v+m/v9sb2cyonNhksQgCbEzlTT2uNiZwobypXnCOg5IqgxtO92MuwR8vJwv3ePEIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + ); + const evalDelta = algosdk.decodeMsgpack( + encodedEvalDelta, + algosdk.EvalDelta + ); + const invalidUtf8 = algosdk.base64ToBytes('/v8='); + const expectedEvalDelta = new algosdk.EvalDelta({ + globalDelta: new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('g1'), invalidUtf8), + new algosdk.ValueDelta({ + action: 1, + uint: BigInt(0), + bytes: utils.concatArrays( + algosdk.coerceToBytes('v1'), + invalidUtf8 + ), + }), + ], + [ + utils.concatArrays(algosdk.coerceToBytes('g2'), invalidUtf8), + new algosdk.ValueDelta({ + action: 2, + uint: BigInt(50), + bytes: new Uint8Array(), + }), + ], + ]), + localDeltas: new Map>([ + [ + 0, + new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('l1'), invalidUtf8), + new algosdk.ValueDelta({ + action: 1, + uint: BigInt(0), + bytes: utils.concatArrays( + algosdk.coerceToBytes('v2'), + invalidUtf8 + ), + }), + ], + ]), + ], + [ + 2, + new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('l2'), invalidUtf8), + new algosdk.ValueDelta({ + action: 2, + uint: BigInt(51), + bytes: new Uint8Array(), + }), + ], + ]), + ], + ]), + sharedAccts: [ + algosdk.Address.fromString( + 'BGYTHFJU624NRGOCQ3ZKK6OCHIHERKQMNU553DF3AR6LZHBP3XR5JLNCUI' + ), + algosdk.Address.zeroAddress(), + ], + logs: [ + utils.concatArrays(algosdk.coerceToBytes('log1'), invalidUtf8), + utils.concatArrays(invalidUtf8, algosdk.coerceToBytes('log2')), + ], + innerTxns: [ + new algosdk.SignedTxnWithAD({ + signedTxn: new algosdk.SignedTransaction({ + txn: new algosdk.Transaction({ + sender: new algosdk.Address( + algosdk.base64ToBytes( + 'f5P+Q6gqfj9Vlv/NIZLnLz9p3pbk2+Di+8o0EJwMZ+k=' + ) + ), + type: algosdk.TransactionType.appl, + suggestedParams: { + flatFee: true, + fee: BigInt(1000), + firstValid: BigInt(92), + lastValid: BigInt(1092), + minFee: BigInt(1000), + }, + appCallParams: { + appIndex: BigInt(7777), + onComplete: algosdk.OnApplicationComplete.NoOpOC, + }, + }), + }), + applyData: new algosdk.ApplyData({ + evalDelta: new algosdk.EvalDelta({ + logs: [ + utils.concatArrays( + algosdk.coerceToBytes('log3'), + invalidUtf8 + ), + utils.concatArrays( + invalidUtf8, + algosdk.coerceToBytes('log4') + ), + ], + }), + }), + }), + ], + }); + assert.deepStrictEqual(evalDelta, expectedEvalDelta); + const reencoded = algosdk.encodeMsgpack(evalDelta); + assert.deepStrictEqual(reencoded, encodedEvalDelta); + }); }); }); From d6e067b3bca3423ce272cd2ec6b223a3275f6c02 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Mon, 29 Jul 2024 11:34:45 -0400 Subject: [PATCH 15/21] Update docs and migration guide --- examples/app.ts | 39 +++++++++++++++++++++-------- src/encoding/binarydata.ts | 2 +- src/encoding/encoding.ts | 29 ++++++++++++++++++--- src/encoding/schema/binarystring.ts | 2 +- src/encoding/schema/map.ts | 2 +- src/main.ts | 1 + tests/2.Encoding.ts | 2 +- v2_TO_v3_MIGRATION_GUIDE.md | 19 +++++++++++++- 8 files changed, 76 insertions(+), 20 deletions(-) diff --git a/examples/app.ts b/examples/app.ts index 0c93150a3..1fa44de72 100644 --- a/examples/app.ts +++ b/examples/app.ts @@ -132,27 +132,44 @@ async function main() { // example: APP_READ_STATE const appInfo = await algodClient.getApplicationByID(appId).do(); - const globalState = appInfo.params.globalState![0]; - console.log(`Raw global state - ${algosdk.encodeJSON(globalState)}`); + if (!appInfo.params.globalState || appInfo.params.globalState.length === 0) { + throw new Error('Global state not present'); + } + const { globalState } = appInfo.params; + console.log( + `Raw global state - ${globalState.map((kv) => algosdk.encodeJSON(kv))}` + ); - const globalKey = globalState.key; + const globalKey = algosdk.base64ToBytes(globalState[0].key); // show global value - const globalValue = globalState.value.bytes; + const globalValue = algosdk.base64ToBytes(globalState[0].value.bytes); - console.log(`Decoded global state - ${globalKey}: ${globalValue}`); + console.log( + `Decoded global state - ${algosdk.bytesToBase64(globalKey)}: ${algosdk.bytesToBase64(globalValue)}` + ); const accountAppInfo = await algodClient .accountApplicationInformation(caller.addr, appId) .do(); + if ( + !accountAppInfo.appLocalState || + !accountAppInfo.appLocalState.keyValue || + accountAppInfo.appLocalState.keyValue.length === 0 + ) { + throw new Error('Local state values not present'); + } + const localState = accountAppInfo.appLocalState.keyValue; + console.log( + `Raw local state - ${localState.map((kv) => algosdk.encodeJSON(kv))}` + ); - const localState = accountAppInfo.appLocalState!.keyValue![0]; - console.log(`Raw local state - ${algosdk.stringifyJSON(localState)}`); - - const localKey = localState.key; + const localKey = algosdk.base64ToBytes(localState[0].key); // get uint value directly - const localValue = localState.value.uint; + const localValue = localState[0].value.uint; - console.log(`Decoded local state - ${localKey}: ${localValue}`); + console.log( + `Decoded local state - ${algosdk.bytesToBase64(localKey)}: ${localValue}` + ); // example: APP_READ_STATE // example: APP_CLOSEOUT diff --git a/src/encoding/binarydata.ts b/src/encoding/binarydata.ts index 7fd09f457..1a6e726fe 100644 --- a/src/encoding/binarydata.ts +++ b/src/encoding/binarydata.ts @@ -29,7 +29,7 @@ export function bytesToBase64(byteArray: Uint8Array): string { } /** - * Decode a base64 string for Node.js and browser environments. + * Convert a byte array to a UTF-8 string. Warning: not all byte arrays are valid UTF-8. * @returns A decoded string */ export function bytesToString(byteArray: Uint8Array): string { diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index f47f08e03..86943c86c 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -126,7 +126,7 @@ export function decodeObj(o: ArrayLike) { * @param options - Options for decoding, including int decoding mode. See {@link IntDecoding} for more information. * @returns The decoded Map object */ -export function decodeAsMap( +export function msgpackRawDecodeAsMap( encoded: ArrayLike, options?: { intDecoding: IntDecoding } ) { @@ -139,7 +139,7 @@ export function decodeAsMap( return msgpackDecode(encoded, decoderOptions); } -function decodeAsMapWithRawStrings( +function msgpackRawDecodeAsMapWithRawStrings( encoded: ArrayLike, options?: { intDecoding: IntDecoding } ) { @@ -241,6 +241,9 @@ interface MsgpackObjectPathSegment { key: string | number | bigint | Uint8Array | RawBinaryString; } +/** + * This class is used to index into an encoded msgpack object and extract raw strings. + */ export class MsgpackRawStringProvider { // eslint-disable-next-line no-use-before-define private readonly parent?: MsgpackRawStringProvider; @@ -272,6 +275,9 @@ export class MsgpackRawStringProvider { this.baseObjectBytes = baseObjectBytes; } + /** + * Create a new provider that resolves to the current provider's map value at the given key. + */ public withMapValue( key: string | number | bigint | Uint8Array | RawBinaryString ): MsgpackRawStringProvider { @@ -284,6 +290,9 @@ export class MsgpackRawStringProvider { }); } + /** + * Create a new provider that resolves to the current provider's array element at the given index. + */ public withArrayElement(index: number): MsgpackRawStringProvider { return new MsgpackRawStringProvider({ parent: this, @@ -294,6 +303,9 @@ export class MsgpackRawStringProvider { }); } + /** + * Get the raw string at the current location. If the current location is not a raw string, an error is thrown. + */ public getRawStringAtCurrentLocation(): Uint8Array { const resolved = this.resolve(); if (resolved instanceof RawBinaryString) { @@ -305,6 +317,9 @@ export class MsgpackRawStringProvider { ); } + /** + * Get the raw string map keys and values at the current location. If the current location is not a map, an error is thrown. + */ public getRawStringKeysAndValuesAtCurrentLocation(): Map< Uint8Array, MsgpackEncodingData @@ -329,6 +344,9 @@ export class MsgpackRawStringProvider { return keysAndValues; } + /** + * Resolve the provider by extracting the value it indicates from the base msgpack object. + */ private resolve(): MsgpackEncodingData { if (this.resolvedCachePresent) { return this.resolvedCache; @@ -338,7 +356,7 @@ export class MsgpackRawStringProvider { parentResolved = this.parent.resolve(); } else { // Need to parse baseObjectBytes - parentResolved = decodeAsMapWithRawStrings( + parentResolved = msgpackRawDecodeAsMapWithRawStrings( this.baseObjectBytes! ) as MsgpackEncodingData; } @@ -402,6 +420,9 @@ export class MsgpackRawStringProvider { throw new Error(`Invalid segment kind: ${this.segment.kind}`); } + /** + * Get the path string of the current location indicated by the provider. Useful for debugging. + */ public getPathString(): string { const parentPathString = this.parent ? this.parent.getPathString() : 'root'; if (!this.segment) { @@ -508,7 +529,7 @@ export function decodeMsgpack( encoded: ArrayLike, c: EncodableClass ): T { - const decoded = decodeAsMap(encoded) as MsgpackEncodingData; + const decoded = msgpackRawDecodeAsMap(encoded) as MsgpackEncodingData; const rawStringProvider = new MsgpackRawStringProvider({ baseObjectBytes: encoded, }); diff --git a/src/encoding/schema/binarystring.ts b/src/encoding/schema/binarystring.ts index a7d3c37a8..d834aaf92 100644 --- a/src/encoding/schema/binarystring.ts +++ b/src/encoding/schema/binarystring.ts @@ -29,7 +29,7 @@ export class SpecialCaseBinaryStringSchema extends Schema { public prepareMsgpack(data: unknown): MsgpackEncodingData { if (data instanceof Uint8Array) { - // TODO: fix cast? + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData return new RawBinaryString(data) as unknown as MsgpackEncodingData; } throw new Error(`Invalid byte array: (${typeof data}) ${data}`); diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 83e788a0b..95f16d3f9 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -513,7 +513,7 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { this.valueSchema.prepareMsgpack(value) ); } - // TODO: fix cast? + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData return prepared as unknown as Map; } diff --git a/src/main.ts b/src/main.ts index 7488baba9..e0d9779b9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -97,6 +97,7 @@ export { decodeObj, msgpackRawEncode, msgpackRawDecode, + msgpackRawDecodeAsMap, encodeMsgpack, decodeMsgpack, encodeJSON, diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index 3c8d12eaf..a2ea544b6 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -973,7 +973,7 @@ describe('encoding', () => { schema: new SpecialCaseBinaryStringSchema(), values: [Uint8Array.from([]), Uint8Array.from([97, 98, 99])], preparedMsgpackValues: [ - // TODO: fix cast? + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData new RawBinaryString( Uint8Array.from([]) ) as unknown as algosdk.MsgpackEncodingData, diff --git a/v2_TO_v3_MIGRATION_GUIDE.md b/v2_TO_v3_MIGRATION_GUIDE.md index 8d6b305cd..722fad8c5 100644 --- a/v2_TO_v3_MIGRATION_GUIDE.md +++ b/v2_TO_v3_MIGRATION_GUIDE.md @@ -303,6 +303,15 @@ In order to facilitate `bigint` as a first-class type in this SDK, additional JS If your v2 code uses `JSON.parse` or `JSON.stringify` on types which can now contain `bigint`s in v3, such as `Transaction` representations or REST API responses, consider using these new functions instead. +### Msgpack Operations + +The functions `encodeObj` and `decodeObj`, used to encode and decode msgpack objects, have been deprecated in v3 in favor of new functions, `msgpackRawEncode` and `msgpackRawDecode`. These functions have clearer names and differ slightly from the old functions. Specifically: + +- `msgpackRawEncode` will encode an object to msgpack, but will not check for empty values and throw errors if any are found. This additional check has become unnecessary due to the new encoding and decoding system in v3. +- `msgpackRawDecode` will decode a msgpack object, but unlike `decodeObj` which always uses `IntDecoding.MIXED` to decode integers, `msgpackRawDecode` can use any provided `IntDecoding` option. If none are provided, it will default to `IntDecoding.BIGINT`. Generally speaking, `IntDecoding.BIGINT` is preferred because it can handle all possible integer values, and the type of an integer will not change depending on the value (like it can with `IntDecoding.MIXED`), meaning code which query integer values from the decoded object will be more predictable. + +Though in the vast majority of cases, you will not need to use these functions directly. Instead, the `encodeMsgpack` and `decodeMsgpack` functions are preferred, which are discussed in the [Object Encoding and Decoding](#object-encoding-and-decoding) section. + ### IntDecoding The `IntDecoding.DEFAULT` option has been renamed to `IntDecoding.UNSAFE` in v3. It behaves identically to the v2 `IntDecoding.DEFAULT` option, but the name has been changed to better reflect the fact that other options should be preferred. @@ -315,7 +324,7 @@ Specifically, the `DryrunResult` class and its dependent types have been removed The `DryrunTransactionResult` class, which made up the elements of the v2 `DryrunResult.txns` array, used to have methods `appTrace` and `lsigTrace`. These have been replaced by the new `dryrunTxnResultAppTrace` and `dryrunTxnResultLogicSigTrace` functions, which accept a `DryrunTxnResult`. These new functions should produce identical results to the old ones. -### Encoding and Decoding +### Object Encoding and Decoding In v2 of the SDK, the `Transaction`, `LogicSig`, `BaseModel` and other classes had `get_obj_for_encoding` methods and `from_obj_for_encoding` static methods. These were used during the process of encoding or decoding objects from msgpack or JSON. These ad-hoc methods have been removed in v3, and in their place a new `Encodable` interface has been introduced, along with functions `encodeMsgpack`, `decodeMsgpack`, `encodeJSON`, and `decodeJSON`. @@ -338,3 +347,11 @@ const encoded = algosdk.encodeMsgpack(txn); // Uint8Array of msgpack-encoded tra const decoded = algosdk.decodeMsgpack(encoded, algosdk.Transaction); // Decoded Transaction instance assert.deepStrictEqual(txn, decoded); ``` + +### Base64 Encoding + +The `base64ToString` function has been removed in v3. Instead, you may combine the `base64ToBytes` and `bytesToString` to achieve the same thing, like so: + +```typescript +algosdk.bytesToString(algosdk.base64ToBytes('SGVsbG8gV29ybGQ=')); +``` From 32c594076822b64e729f935fc96d2d9bb18643da Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 31 Jul 2024 10:24:56 -0400 Subject: [PATCH 16/21] CR changes --- src/encoding/schema/map.ts | 17 ++++++++++++----- src/group.ts | 9 ++++++--- tests/5.Transaction.ts | 2 +- tests/7.AlgoSDK.ts | 12 ++++++++++-- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 95f16d3f9..43db92e61 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -450,7 +450,14 @@ export class StringMapSchema extends Schema { } } -function removeRawStringsFromMsgpackValues( +/** + * Converts any RawBinaryString values to regular strings in a MsgpackEncodingData object. + * + * Note this conversion may be lossy if the binary data is not valid UTF-8. + * + * @returns A new object with RawBinaryString values converted to strings. + */ +function convertRawStringsInMsgpackValue( value: MsgpackEncodingData ): MsgpackEncodingData { if (value instanceof RawBinaryString) { @@ -463,18 +470,18 @@ function removeRawStringsFromMsgpackValues( >(); for (const [key, val] of value) { newMap.set( - removeRawStringsFromMsgpackValues(key) as + convertRawStringsInMsgpackValue(key) as | string | number | bigint | Uint8Array, - removeRawStringsFromMsgpackValues(val) + convertRawStringsInMsgpackValue(val) ); } return newMap; } if (Array.isArray(value)) { - return value.map(removeRawStringsFromMsgpackValues); + return value.map(convertRawStringsInMsgpackValue); } return value; } @@ -528,7 +535,7 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { map.set( key, this.valueSchema.fromPreparedMsgpack( - removeRawStringsFromMsgpackValues(value), + convertRawStringsInMsgpackValue(value), rawStringProvider.withMapValue(new RawBinaryString(key)) ) ); diff --git a/src/group.ts b/src/group.ts index f870f518d..4b0318e2b 100644 --- a/src/group.ts +++ b/src/group.ts @@ -1,6 +1,6 @@ import { Transaction } from './transaction.js'; import * as nacl from './nacl/naclWrappers.js'; -import { encodeObj } from './encoding/encoding.js'; +import { msgpackRawEncode } from './encoding/encoding.js'; import * as utils from './utils/utils.js'; const ALGORAND_MAX_TX_GROUP_SIZE = 16; @@ -8,11 +8,14 @@ const TX_GROUP_TAG = new TextEncoder().encode('TG'); function txGroupPreimage(txnHashes: Uint8Array[]): Uint8Array { if (txnHashes.length > ALGORAND_MAX_TX_GROUP_SIZE) { - throw Error( + throw new Error( `${txnHashes.length} transactions grouped together but max group size is ${ALGORAND_MAX_TX_GROUP_SIZE}` ); } - const bytes = encodeObj({ + if (txnHashes.length === 0) { + throw new Error('Cannot compute group ID of zero transactions'); + } + const bytes = msgpackRawEncode({ txlist: txnHashes, }); return utils.concatArrays(TX_GROUP_TAG, bytes); diff --git a/tests/5.Transaction.ts b/tests/5.Transaction.ts index f329a50ac..a516a1466 100644 --- a/tests/5.Transaction.ts +++ b/tests/5.Transaction.ts @@ -1064,7 +1064,7 @@ describe('Sign', () => { ); assert.ok(encRep instanceof Map && !encRep.has('fadd')); - const encTxn = algosdk.encodeObj(encRep); + const encTxn = algosdk.msgpackRawEncode(encRep); const golden = algosdk.base64ToBytes( 'iqRhZnJ6w6RmYWlkzScPo2ZlZc0IeqJmdgGjZ2VurG1vY2stbmV0d29ya6JnaMQgSGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiKibHbNA+mkbm90ZcQDewzIo3NuZMQgoImqaSLjuZj63/bNSAjd+eAh5JROOJ6j1cY4eGaJGX6kdHlwZaRhZnJ6' ); diff --git a/tests/7.AlgoSDK.ts b/tests/7.AlgoSDK.ts index 863d9ccee..d51df817a 100644 --- a/tests/7.AlgoSDK.ts +++ b/tests/7.AlgoSDK.ts @@ -20,12 +20,20 @@ describe('Algosdk (AKA end to end)', () => { describe('#encoding', () => { it('should encode and decode', () => { const o = { a: [1, 2, 3, 4, 5], b: 3486, c: 'skfg' }; - assert.deepStrictEqual(o, algosdk.decodeObj(algosdk.encodeObj(o))); + assert.deepStrictEqual( + o, + algosdk.msgpackRawDecode(algosdk.msgpackRawEncode(o), { + intDecoding: algosdk.IntDecoding.MIXED, + }) + ); }); it('should encode and decode strings', () => { const o = 'Hi there'; - assert.deepStrictEqual(o, algosdk.decodeObj(algosdk.encodeObj(o as any))); + assert.deepStrictEqual( + o, + algosdk.msgpackRawDecode(algosdk.msgpackRawEncode(o)) + ); }); it('should not mutate unsigned transaction when going to or from encoded buffer', () => { From 6fb6e799a5bb99bb4982d3d7dad3df2372da92b4 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 31 Jul 2024 11:42:33 -0400 Subject: [PATCH 17/21] Update to latest lib commit --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index fcc2372f1..896566c0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1197,7 +1197,7 @@ "node_modules/algorand-msgpack": { "version": "1.0.1", "resolved": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", - "integrity": "sha512-lqopii/JLsD1EmKpOG0Qq3itPzhb7xBkR3P7ClNJicpld06oeQMud3YnmntjDmqxMl/r38Q2T+cFXFdEg6P9xw==", + "integrity": "sha512-tQxNjVpx2Hm6ASKkJeOXPBSjpB6v8dIBfg5CSUleC9i+Xt0qf3nyUKPpWMjz7yDC+lYYVPNSZWnnxbLEPYmOtQ==", "license": "ISC", "engines": { "node": ">= 14" From 800d64ed1474c1f22e82c44b183719fd44900d9f Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 31 Jul 2024 15:33:36 -0400 Subject: [PATCH 18/21] Use published algorand-msgpack --- .circleci/config.yml | 12 ------------ package-lock.json | 9 ++++----- package.json | 2 +- tests/cucumber/docker/Dockerfile | 10 +++------- 4 files changed, 8 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d2343173..554e293af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,21 +138,9 @@ commands: << parameters.sudo >> apt -y install curl make git build-essential jq unzip - node/install: node-version: '18' - - run: - name: Install algorand-msgpack from branch (temporary) - command: | - set -e - cd .. - git clone https://github.com/algorand/msgpack-javascript.git - cd msgpack-javascript - git checkout raw-string-encoding - npm i - npm run prepare - npm pack - run: name: npm ci command: | set -e npm ci - cp ../msgpack-javascript/algorand-msgpack-1.0.1.tgz algorand-msgpack-1.0.1.tgz if [ "<< parameters.browser >>" == "chrome" ]; then npm install chromedriver@latest; fi diff --git a/package-lock.json b/package-lock.json index 896566c0d..cd7733d46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "2.8.0", "license": "MIT", "dependencies": { - "algorand-msgpack": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", + "algorand-msgpack": "^1.1.0", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", @@ -1195,10 +1195,9 @@ } }, "node_modules/algorand-msgpack": { - "version": "1.0.1", - "resolved": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", - "integrity": "sha512-tQxNjVpx2Hm6ASKkJeOXPBSjpB6v8dIBfg5CSUleC9i+Xt0qf3nyUKPpWMjz7yDC+lYYVPNSZWnnxbLEPYmOtQ==", - "license": "ISC", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/algorand-msgpack/-/algorand-msgpack-1.1.0.tgz", + "integrity": "sha512-08k7pBQnkaUB5p+jL7f1TRaUIlTSDE0cesFu1mD7llLao+1cAhtvvZmGE3OnisTd0xOn118QMw74SRqddqaYvw==", "engines": { "node": ">= 14" } diff --git a/package.json b/package.json index 6a64daff0..aaf31fa31 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "url": "git://github.com/algorand/js-algorand-sdk.git" }, "dependencies": { - "algorand-msgpack": "file:../msgpack-javascript/algorand-msgpack-1.0.1.tgz", + "algorand-msgpack": "^1.1.0", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", diff --git a/tests/cucumber/docker/Dockerfile b/tests/cucumber/docker/Dockerfile index 43f7d2e65..4ef89a836 100644 --- a/tests/cucumber/docker/Dockerfile +++ b/tests/cucumber/docker/Dockerfile @@ -18,13 +18,9 @@ RUN wget -q -O - https://deb.nodesource.com/setup_18.x | bash \ && echo "npm version: $(npm --version)" # Copy SDK code into the container -RUN mkdir -p /home/js-algorand-sdk -COPY . /home/js-algorand-sdk -WORKDIR /home/js-algorand-sdk - -# Temporary hack to copy in msgpack-javascript from a branch -RUN mkdir -p /home/msgpack-javascript -COPY algorand-msgpack-1.0.1.tgz /home/msgpack-javascript/algorand-msgpack-1.0.1.tgz +RUN mkdir -p $HOME/js-algorand-sdk +COPY . $HOME/js-algorand-sdk +WORKDIR $HOME/js-algorand-sdk ARG TEST_BROWSER ENV TEST_BROWSER=$TEST_BROWSER From fa942d2414aa47b58a1098a90231376ccd4c535b Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Fri, 2 Aug 2024 16:56:32 -0400 Subject: [PATCH 19/21] Introduce options to error on lossy string conversion --- src/encoding/encoding.ts | 41 +++++++++-- src/encoding/schema/address.ts | 7 +- src/encoding/schema/array.ts | 8 ++- src/encoding/schema/binarystring.ts | 22 ++++-- src/encoding/schema/boolean.ts | 6 +- src/encoding/schema/bytearray.ts | 7 +- src/encoding/schema/map.ts | 47 ++++++++++--- src/encoding/schema/optional.ts | 8 ++- src/encoding/schema/string.ts | 7 +- src/encoding/schema/uint64.ts | 7 +- src/encoding/schema/untyped.ts | 7 +- tests/2.Encoding.ts | 103 +++++++++++++++++++++++++--- 12 files changed, 233 insertions(+), 37 deletions(-) diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 86943c86c..65327ae10 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -438,6 +438,18 @@ export class MsgpackRawStringProvider { } } +/** + * Options for {@link Schema.prepareJSON} + */ +export interface PrepareJSONOptions { + /** + * If true, throw an error if a string is encountered that cannot be represented as a UTF-8 JSON string. + * + * Otherwise, such a string will be encoded without error but may lose information. + */ + strictBinaryStrings?: boolean; +} + /** * A Schema is used to prepare objects for encoding and decoding from msgpack and JSON. * @@ -479,7 +491,10 @@ export abstract class Schema { * @param data - The JSON encoding data to be prepared. * @returns A value ready to be JSON encoded. */ - public abstract prepareJSON(data: unknown): JSONEncodingData; + public abstract prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData; /** * Restores the encoding data from a JSON encoding object. @@ -567,13 +582,31 @@ export function decodeJSON( ); } +export interface EncodeJSONOptions { + /** + * Adds indentation, white space, and line break characters to the return-value JSON text to make + * it easier to read. + */ + space?: string | number; + + /** + * If true, throw an error if a string is encountered that cannot be represented as a UTF-8 JSON string. + * + * Otherwise, such a string will be encoded without error but may lose information. + */ + strictBinaryStrings?: boolean; +} + /** * Encode an Encodable object to a JSON string. * @param e - The object to encode - * @param space - Adds indentation, white space, and line break characters to the return-value JSON text to make it easier to read. + * @param options - Optional encoding options. See {@link EncodeJSONOptions} for more information. * @returns A JSON string encoding of the object */ -export function encodeJSON(e: Encodable, space?: string | number): string { - const prepared = e.getEncodingSchema().prepareJSON(e.toEncodingData()); +export function encodeJSON(e: Encodable, options?: EncodeJSONOptions): string { + const { space, ...prepareJSONOptions } = options ?? {}; + const prepared = e + .getEncodingSchema() + .prepareJSON(e.toEncodingData(), prepareJSONOptions); return stringifyJSON(prepared, undefined, space); } diff --git a/src/encoding/schema/address.ts b/src/encoding/schema/address.ts index 6de59fb1b..3eb7f5c7e 100644 --- a/src/encoding/schema/address.ts +++ b/src/encoding/schema/address.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; import { Address } from '../address.js'; @@ -34,7 +35,11 @@ export class AddressSchema extends Schema { return new Address(encoded as Uint8Array); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (data instanceof Address) { return data.toString(); } diff --git a/src/encoding/schema/array.ts b/src/encoding/schema/array.ts index c47995dc7..7306f7c26 100644 --- a/src/encoding/schema/array.ts +++ b/src/encoding/schema/array.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -42,9 +43,12 @@ export class ArraySchema extends Schema { throw new Error('ArraySchema encoded data must be an array'); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (Array.isArray(data)) { - return data.map((item) => this.itemSchema.prepareJSON(item)); + return data.map((item) => this.itemSchema.prepareJSON(item, options)); } throw new Error('ArraySchema data must be an array'); } diff --git a/src/encoding/schema/binarystring.ts b/src/encoding/schema/binarystring.ts index d834aaf92..bf161c373 100644 --- a/src/encoding/schema/binarystring.ts +++ b/src/encoding/schema/binarystring.ts @@ -4,8 +4,10 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; -import { coerceToBytes, bytesToString } from '../binarydata.js'; +import { coerceToBytes, bytesToString, bytesToBase64 } from '../binarydata.js'; +import { arrayEqual } from '../../utils/utils.js'; /* eslint-disable class-methods-use-this */ @@ -42,10 +44,22 @@ export class SpecialCaseBinaryStringSchema extends Schema { return rawStringProvider.getRawStringAtCurrentLocation(); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (data instanceof Uint8Array) { - // WARNING: not safe for all binary data - return bytesToString(data); + // Not safe to convert to string for all binary data + const stringValue = bytesToString(data); + if ( + options.strictBinaryStrings && + !arrayEqual(coerceToBytes(stringValue), data) + ) { + throw new Error( + `Invalid UTF-8 byte array encountered with strictBinaryStrings enabled. Base64 value: ${bytesToBase64(data)}` + ); + } + return stringValue; } throw new Error(`Invalid byte array: (${typeof data}) ${data}`); } diff --git a/src/encoding/schema/boolean.ts b/src/encoding/schema/boolean.ts index 17bb53475..090a67545 100644 --- a/src/encoding/schema/boolean.ts +++ b/src/encoding/schema/boolean.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -34,7 +35,10 @@ export class BooleanSchema extends Schema { throw new Error('Invalid boolean'); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (typeof data === 'boolean') { return data; } diff --git a/src/encoding/schema/bytearray.ts b/src/encoding/schema/bytearray.ts index a063203d0..5218049ca 100644 --- a/src/encoding/schema/bytearray.ts +++ b/src/encoding/schema/bytearray.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; import { base64ToBytes, bytesToBase64 } from '../binarydata.js'; @@ -35,7 +36,11 @@ export class ByteArraySchema extends Schema { throw new Error(`Invalid byte array: (${typeof encoded}) ${encoded}`); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (data instanceof Uint8Array) { return bytesToBase64(data); } diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 43db92e61..0d2c99b5d 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -4,9 +4,10 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; -import { ensureUint64 } from '../../utils/utils.js'; -import { bytesToString, coerceToBytes } from '../binarydata.js'; +import { ensureUint64, arrayEqual } from '../../utils/utils.js'; +import { bytesToString, coerceToBytes, bytesToBase64 } from '../binarydata.js'; /* eslint-disable class-methods-use-this */ @@ -171,7 +172,10 @@ export class NamedMapSchema extends Schema { return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error('NamedMapSchema data must be a Map'); } @@ -181,7 +185,7 @@ export class NamedMapSchema extends Schema { if (entry.omitEmpty && entry.valueSchema.isDefaultValue(value)) { continue; } - obj[entry.key] = entry.valueSchema.prepareJSON(value); + obj[entry.key] = entry.valueSchema.prepareJSON(value, options); } return obj; } @@ -304,7 +308,10 @@ export class Uint64MapSchema extends Schema { return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error( `Uint64MapSchema data must be a Map. Got (${typeof data}) ${data}` @@ -316,7 +323,7 @@ export class Uint64MapSchema extends Schema { if (prepared.has(bigintKey)) { throw new Error(`Duplicate key: ${bigintKey}`); } - prepared.set(bigintKey, this.valueSchema.prepareJSON(value)); + prepared.set(bigintKey, this.valueSchema.prepareJSON(value, options)); } // Convert map to object const obj: { [key: string]: JSONEncodingData } = {}; @@ -407,7 +414,10 @@ export class StringMapSchema extends Schema { return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error( `StringMapSchema data must be a Map. Got (${typeof data}) ${data}` @@ -421,7 +431,7 @@ export class StringMapSchema extends Schema { if (prepared.has(key)) { throw new Error(`Duplicate key: ${key}`); } - prepared.set(key, this.valueSchema.prepareJSON(value)); + prepared.set(key, this.valueSchema.prepareJSON(value, options)); } // Convert map to object const obj: { [key: string]: JSONEncodingData } = {}; @@ -543,7 +553,10 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error( `SpecialCaseBinaryStringMapSchema data must be a Map. Got (${typeof data}) ${data}` @@ -554,8 +567,20 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { if (!(key instanceof Uint8Array)) { throw new Error(`Invalid key: ${key}`); } - // WARNING: not safe for all binary data - prepared.set(bytesToString(key), this.valueSchema.prepareJSON(value)); + // Not safe to convert to string for all binary data + const keyStringValue = bytesToString(key); + if ( + options.strictBinaryStrings && + !arrayEqual(coerceToBytes(keyStringValue), key) + ) { + throw new Error( + `Invalid UTF-8 byte array encountered with strictBinaryStrings enabled. Base64 value: ${bytesToBase64(key)}` + ); + } + prepared.set( + keyStringValue, + this.valueSchema.prepareJSON(value, options) + ); } // Convert map to object const obj: { [key: string]: JSONEncodingData } = {}; diff --git a/src/encoding/schema/optional.ts b/src/encoding/schema/optional.ts index 96ff805c4..b083cebeb 100644 --- a/src/encoding/schema/optional.ts +++ b/src/encoding/schema/optional.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -49,12 +50,15 @@ export class OptionalSchema extends Schema { return this.valueSchema.fromPreparedMsgpack(encoded, rawStringProvider); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (data === undefined) { // JSON representation does not have undefined, only null return null; } - return this.valueSchema.prepareJSON(data); + return this.valueSchema.prepareJSON(data, options); } public fromPreparedJSON(encoded: JSONEncodingData): unknown { diff --git a/src/encoding/schema/string.ts b/src/encoding/schema/string.ts index be8be78c5..4f0081fe4 100644 --- a/src/encoding/schema/string.ts +++ b/src/encoding/schema/string.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -34,7 +35,11 @@ export class StringSchema extends Schema { throw new Error(`Invalid string: (${typeof encoded}) ${encoded}`); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (typeof data === 'string') { return data; } diff --git a/src/encoding/schema/uint64.ts b/src/encoding/schema/uint64.ts index 4f2cd29bb..81f1dbbc5 100644 --- a/src/encoding/schema/uint64.ts +++ b/src/encoding/schema/uint64.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, } from '../encoding.js'; import { ensureUint64 } from '../../utils/utils.js'; @@ -31,7 +32,11 @@ export class Uint64Schema extends Schema { return ensureUint64(encoded); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { return ensureUint64(data); } diff --git a/src/encoding/schema/untyped.ts b/src/encoding/schema/untyped.ts index 1ee5ecbdb..6551d6159 100644 --- a/src/encoding/schema/untyped.ts +++ b/src/encoding/schema/untyped.ts @@ -3,6 +3,7 @@ import { MsgpackEncodingData, MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, msgpackEncodingDataToJSONEncodingData, jsonEncodingDataToMsgpackEncodingData, } from '../encoding.js'; @@ -32,7 +33,11 @@ export class UntypedSchema extends Schema { return encoded; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { return msgpackEncodingDataToJSONEncodingData(data as MsgpackEncodingData); } diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index a2ea544b6..ab406018b 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -1,7 +1,7 @@ /* eslint-env mocha */ import assert from 'assert'; import { RawBinaryString } from 'algorand-msgpack'; -import algosdk from '../src/index.js'; +import algosdk, { bytesToString, coerceToBytes } from '../src/index.js'; import * as utils from '../src/utils/utils.js'; import { Schema, MsgpackRawStringProvider } from '../src/encoding/encoding.js'; import { @@ -1354,7 +1354,7 @@ describe('encoding', () => { roundtripMsgpackExpectedValue ); - const actualJson = testcase.schema.prepareJSON(value); + const actualJson = testcase.schema.prepareJSON(value, {}); assert.deepStrictEqual(actualJson, preparedJsonValue); const roundtripJsonValue = @@ -1558,7 +1558,7 @@ describe('encoding', () => { allEmptyValuesRestored ); - let prepareJsonResult = schema.prepareJSON(allEmptyValues); + let prepareJsonResult = schema.prepareJSON(allEmptyValues, {}); // All empty values should be omitted assert.deepStrictEqual(prepareJsonResult, {}); let fromPreparedJsonResult = schema.fromPreparedJSON(prepareJsonResult); @@ -1587,7 +1587,7 @@ describe('encoding', () => { // Values are restored properly assert.deepStrictEqual(fromPreparedMsgpackResult, allNonemptyValues); - prepareJsonResult = schema.prepareJSON(allNonemptyValues); + prepareJsonResult = schema.prepareJSON(allNonemptyValues, {}); // All values are present assert.strictEqual( Object.keys(prepareJsonResult as object).length, @@ -1628,7 +1628,8 @@ describe('encoding', () => { ['a', ''], ['b', ''], ['c', ''], - ]) + ]), + {} ), {} ); @@ -1652,7 +1653,8 @@ describe('encoding', () => { ['a', '1'], ['b', '2'], ['c', '3'], - ]) + ]), + {} ), { a: '1', @@ -1694,7 +1696,8 @@ describe('encoding', () => { ['c', ''], ]), ], - ]) + ]), + {} ), {} ); @@ -1733,7 +1736,8 @@ describe('encoding', () => { ['c', '3'], ]), ], - ]) + ]), + {} ), { map: { a: '1', b: '2' }, @@ -1926,6 +1930,89 @@ describe('encoding', () => { ); }); }); + describe('strictBinaryStrings', () => { + const invalidUtf8String = Uint8Array.from([ + 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, + 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, + 135, 121, + ]); + + const invalidUtf8StringEncoded = bytesToString(invalidUtf8String); + const invalidUtf8StringDecoded = coerceToBytes(invalidUtf8StringEncoded); + + it('should have lossy string conversion', () => { + assert.notStrictEqual(invalidUtf8String, invalidUtf8StringDecoded); + }); + + it('should lossily prepare invalid UTF-8 strings by default and when disabled', () => { + for (const options of [{}, { strictBinaryStrings: false }]) { + const schema = new SpecialCaseBinaryStringSchema(); + const prepared = schema.prepareJSON(invalidUtf8String, options); + assert.strictEqual(prepared, invalidUtf8StringEncoded); + assert.deepStrictEqual( + schema.fromPreparedJSON(prepared), + invalidUtf8StringDecoded + ); + + const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); + // testing default behavior + const preparedMap = mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + {} + ); + const expectedPreparedMap: Record = {}; + expectedPreparedMap[invalidUtf8StringEncoded] = + invalidUtf8StringEncoded; + assert.deepStrictEqual(preparedMap, expectedPreparedMap); + assert.deepStrictEqual( + mapSchema.fromPreparedJSON(preparedMap), + new Map([[invalidUtf8StringDecoded, invalidUtf8StringDecoded]]) + ); + } + }); + it('should error when preparing invalid UTF-8 strings when enabled', () => { + const schema = new SpecialCaseBinaryStringSchema(); + assert.throws( + () => + schema.prepareJSON(invalidUtf8String, { + strictBinaryStrings: true, + }), + /Invalid UTF-8 byte array encountered/ + ); + + const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[Uint8Array.from([97]), invalidUtf8String]]), + { + strictBinaryStrings: true, + } + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, Uint8Array.from([97])]]), + { + strictBinaryStrings: true, + } + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + { strictBinaryStrings: true } + ), + /Invalid UTF-8 byte array encountered/ + ); + }); + }); describe('MsgpackRawStringProvider', () => { it('correctly records paths and provides raw strings', () => { const baseObject = new Map([ From 561aa4db47789ebed15c5ab6503c23acc4f1ce20 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Fri, 2 Aug 2024 16:56:46 -0400 Subject: [PATCH 20/21] Swap default behavior --- src/encoding/encoding.ts | 12 +-- src/encoding/schema/binarystring.ts | 4 +- src/encoding/schema/map.ts | 4 +- tests/2.Encoding.ts | 118 ++++++++++++++-------------- 4 files changed, 67 insertions(+), 71 deletions(-) diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 65327ae10..b274411ee 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -443,11 +443,11 @@ export class MsgpackRawStringProvider { */ export interface PrepareJSONOptions { /** - * If true, throw an error if a string is encountered that cannot be represented as a UTF-8 JSON string. + * If true, allows invalid UTF-8 binary strings to be converted to JSON strings. * - * Otherwise, such a string will be encoded without error but may lose information. + * Otherwise, an error will be thrown if encoding a binary string to a JSON cannot be done losslessly. */ - strictBinaryStrings?: boolean; + lossyBinaryStringConversion?: boolean; } /** @@ -590,11 +590,11 @@ export interface EncodeJSONOptions { space?: string | number; /** - * If true, throw an error if a string is encountered that cannot be represented as a UTF-8 JSON string. + * If true, allows invalid UTF-8 binary strings to be converted to JSON strings. * - * Otherwise, such a string will be encoded without error but may lose information. + * Otherwise, an error will be thrown if encoding a binary string to a JSON cannot be done losslessly. */ - strictBinaryStrings?: boolean; + lossyBinaryStringConversion?: boolean; } /** diff --git a/src/encoding/schema/binarystring.ts b/src/encoding/schema/binarystring.ts index bf161c373..01308732b 100644 --- a/src/encoding/schema/binarystring.ts +++ b/src/encoding/schema/binarystring.ts @@ -52,11 +52,11 @@ export class SpecialCaseBinaryStringSchema extends Schema { // Not safe to convert to string for all binary data const stringValue = bytesToString(data); if ( - options.strictBinaryStrings && + !options.lossyBinaryStringConversion && !arrayEqual(coerceToBytes(stringValue), data) ) { throw new Error( - `Invalid UTF-8 byte array encountered with strictBinaryStrings enabled. Base64 value: ${bytesToBase64(data)}` + `Invalid UTF-8 byte array encountered. Encode with lossyBinaryStringConversion enabled to bypass this check. Base64 value: ${bytesToBase64(data)}` ); } return stringValue; diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 0d2c99b5d..f0afee3cb 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -570,11 +570,11 @@ export class SpecialCaseBinaryStringMapSchema extends Schema { // Not safe to convert to string for all binary data const keyStringValue = bytesToString(key); if ( - options.strictBinaryStrings && + !options.lossyBinaryStringConversion && !arrayEqual(coerceToBytes(keyStringValue), key) ) { throw new Error( - `Invalid UTF-8 byte array encountered with strictBinaryStrings enabled. Base64 value: ${bytesToBase64(key)}` + `Invalid UTF-8 byte array encountered. Encode with lossyBinaryStringConversion enabled to bypass this check. Base64 value: ${bytesToBase64(key)}` ); } prepared.set( diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index ab406018b..c3bdbfc91 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -1930,7 +1930,7 @@ describe('encoding', () => { ); }); }); - describe('strictBinaryStrings', () => { + describe('lossyBinaryStringConversion', () => { const invalidUtf8String = Uint8Array.from([ 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, @@ -1940,77 +1940,73 @@ describe('encoding', () => { const invalidUtf8StringEncoded = bytesToString(invalidUtf8String); const invalidUtf8StringDecoded = coerceToBytes(invalidUtf8StringEncoded); - it('should have lossy string conversion', () => { + it('should have lossy string conversion for invalid UTF-8 string', () => { assert.notStrictEqual(invalidUtf8String, invalidUtf8StringDecoded); }); - it('should lossily prepare invalid UTF-8 strings by default and when disabled', () => { - for (const options of [{}, { strictBinaryStrings: false }]) { - const schema = new SpecialCaseBinaryStringSchema(); - const prepared = schema.prepareJSON(invalidUtf8String, options); - assert.strictEqual(prepared, invalidUtf8StringEncoded); - assert.deepStrictEqual( - schema.fromPreparedJSON(prepared), - invalidUtf8StringDecoded - ); + it('should lossily prepare invalid UTF-8 strings by default and when enabled', () => { + const options = { + lossyBinaryStringConversion: true, + }; - const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); - // testing default behavior - const preparedMap = mapSchema.prepareJSON( - new Map([[invalidUtf8String, invalidUtf8String]]), - {} - ); - const expectedPreparedMap: Record = {}; - expectedPreparedMap[invalidUtf8StringEncoded] = - invalidUtf8StringEncoded; - assert.deepStrictEqual(preparedMap, expectedPreparedMap); - assert.deepStrictEqual( - mapSchema.fromPreparedJSON(preparedMap), - new Map([[invalidUtf8StringDecoded, invalidUtf8StringDecoded]]) - ); - } - }); - it('should error when preparing invalid UTF-8 strings when enabled', () => { const schema = new SpecialCaseBinaryStringSchema(); - assert.throws( - () => - schema.prepareJSON(invalidUtf8String, { - strictBinaryStrings: true, - }), - /Invalid UTF-8 byte array encountered/ + const prepared = schema.prepareJSON(invalidUtf8String, options); + assert.strictEqual(prepared, invalidUtf8StringEncoded); + assert.deepStrictEqual( + schema.fromPreparedJSON(prepared), + invalidUtf8StringDecoded ); const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); - assert.throws( - () => - mapSchema.prepareJSON( - new Map([[Uint8Array.from([97]), invalidUtf8String]]), - { - strictBinaryStrings: true, - } - ), - /Invalid UTF-8 byte array encountered/ + const preparedMap = mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + options ); - - assert.throws( - () => - mapSchema.prepareJSON( - new Map([[invalidUtf8String, Uint8Array.from([97])]]), - { - strictBinaryStrings: true, - } - ), - /Invalid UTF-8 byte array encountered/ + const expectedPreparedMap: Record = {}; + expectedPreparedMap[invalidUtf8StringEncoded] = + invalidUtf8StringEncoded; + assert.deepStrictEqual(preparedMap, expectedPreparedMap); + assert.deepStrictEqual( + mapSchema.fromPreparedJSON(preparedMap), + new Map([[invalidUtf8StringDecoded, invalidUtf8StringDecoded]]) ); + }); + it('should error when preparing invalid UTF-8 strings when disabled and by default', () => { + for (const options of [{}, { lossyBinaryStringConversion: false }]) { + const schema = new SpecialCaseBinaryStringSchema(); + assert.throws( + () => schema.prepareJSON(invalidUtf8String, options), + /Invalid UTF-8 byte array encountered/ + ); - assert.throws( - () => - mapSchema.prepareJSON( - new Map([[invalidUtf8String, invalidUtf8String]]), - { strictBinaryStrings: true } - ), - /Invalid UTF-8 byte array encountered/ - ); + const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[Uint8Array.from([97]), invalidUtf8String]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, Uint8Array.from([97])]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + } }); }); describe('MsgpackRawStringProvider', () => { From f560311c0cad0363c91e842fa1e2f41710684f12 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Fri, 2 Aug 2024 17:26:16 -0400 Subject: [PATCH 21/21] Fix mistake --- tests/7.AlgoSDK.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/7.AlgoSDK.ts b/tests/7.AlgoSDK.ts index d51df817a..aec20ad69 100644 --- a/tests/7.AlgoSDK.ts +++ b/tests/7.AlgoSDK.ts @@ -1075,7 +1075,8 @@ describe('Algosdk (AKA end to end)', () => { it('should be properly serialized to JSON', () => { const forEncoding = algosdk.modelsv2.DryrunRequest.encodingSchema.prepareJSON( - req.toEncodingData() + req.toEncodingData(), + {} ); const actual = algosdk.stringifyJSON(forEncoding, undefined, 2);