diff --git a/packages/ripple-binary-codec/HISTORY.md b/packages/ripple-binary-codec/HISTORY.md index c2a02326..15a7636d 100644 --- a/packages/ripple-binary-codec/HISTORY.md +++ b/packages/ripple-binary-codec/HISTORY.md @@ -1,6 +1,8 @@ # ripple-binary-codec Release History ## Unreleased +### Added +- Allow custom type definitions to be used for encoding/decoding transactions at runtime (e.g. for sidechains/new amendments) ## 1.5.0 (2023-03-08) ### Changed diff --git a/packages/ripple-binary-codec/src/binary.ts b/packages/ripple-binary-codec/src/binary.ts index 61fd1dcd..f578f944 100644 --- a/packages/ripple-binary-codec/src/binary.ts +++ b/packages/ripple-binary-codec/src/binary.ts @@ -6,7 +6,11 @@ import { AccountID } from './types/account-id' import { HashPrefix } from './hash-prefixes' import { BinarySerializer, BytesList } from './serdes/binary-serializer' import { sha512Half, transactionID } from './hashes' -import { FieldInstance } from './enums' +import { + type XrplDefinitionsBase, + DEFAULT_DEFINITIONS, + type FieldInstance, +} from './enums' import { STObject } from './types/st-object' import { JsonObject } from './types/serialized-type' import { Buffer } from 'buffer/' @@ -16,26 +20,41 @@ import bigInt = require('big-integer') * Construct a BinaryParser * * @param bytes hex-string to construct BinaryParser from + * @param definitions rippled definitions used to parse the values of transaction types and such. + * Can be customized for sidechains and amendments. * @returns A BinaryParser */ -const makeParser = (bytes: string): BinaryParser => new BinaryParser(bytes) +const makeParser = ( + bytes: string, + definitions?: XrplDefinitionsBase, +): BinaryParser => new BinaryParser(bytes, definitions) /** * Parse BinaryParser into JSON * * @param parser BinaryParser object + * @param definitions rippled definitions used to parse the values of transaction types and such. + * Can be customized for sidechains and amendments. * @returns JSON for the bytes in the BinaryParser */ -const readJSON = (parser: BinaryParser): JsonObject => - (parser.readType(coreTypes.STObject) as STObject).toJSON() +const readJSON = ( + parser: BinaryParser, + definitions: XrplDefinitionsBase = DEFAULT_DEFINITIONS, +): JsonObject => + (parser.readType(coreTypes.STObject) as STObject).toJSON(definitions) /** * Parse a hex-string into its JSON interpretation * * @param bytes hex-string to parse into JSON + * @param definitions rippled definitions used to parse the values of transaction types and such. + * Can be customized for sidechains and amendments. * @returns JSON */ -const binaryToJSON = (bytes: string): JsonObject => readJSON(makeParser(bytes)) +const binaryToJSON = ( + bytes: string, + definitions?: XrplDefinitionsBase, +): JsonObject => readJSON(makeParser(bytes, definitions), definitions) /** * Interface for passing parameters to SerializeObject @@ -46,17 +65,18 @@ interface OptionObject { prefix?: Buffer suffix?: Buffer signingFieldsOnly?: boolean + definitions?: XrplDefinitionsBase } /** * Function to serialize JSON object representing a transaction * * @param object JSON object to serialize - * @param opts options for serializing, including optional prefix, suffix, and signingFieldOnly + * @param opts options for serializing, including optional prefix, suffix, signingFieldOnly, and definitions * @returns A Buffer containing the serialized object */ function serializeObject(object: JsonObject, opts: OptionObject = {}): Buffer { - const { prefix, suffix, signingFieldsOnly = false } = opts + const { prefix, suffix, signingFieldsOnly = false, definitions } = opts const bytesList = new BytesList() if (prefix) { @@ -66,8 +86,9 @@ function serializeObject(object: JsonObject, opts: OptionObject = {}): Buffer { const filter = signingFieldsOnly ? (f: FieldInstance): boolean => f.isSigningField : undefined - - coreTypes.STObject.from(object, filter).toBytesSink(bytesList) + ;(coreTypes.STObject as typeof STObject) + .from(object, filter, definitions) + .toBytesSink(bytesList) if (suffix) { bytesList.put(suffix) @@ -81,13 +102,19 @@ function serializeObject(object: JsonObject, opts: OptionObject = {}): Buffer { * * @param transaction Transaction to serialize * @param prefix Prefix bytes to put before the serialized object + * @param opts.definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns A Buffer with the serialized object */ function signingData( transaction: JsonObject, prefix: Buffer = HashPrefix.transactionSig, + opts: { definitions?: XrplDefinitionsBase } = {}, ): Buffer { - return serializeObject(transaction, { prefix, signingFieldsOnly: true }) + return serializeObject(transaction, { + prefix, + signingFieldsOnly: true, + definitions: opts.definitions, + }) } /** @@ -102,6 +129,7 @@ interface ClaimObject extends JsonObject { * Serialize a signingClaim * * @param claim A claim object to serialize + * @param opts.definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns the serialized object with appropriate prefix */ function signingClaimData(claim: ClaimObject): Buffer { @@ -123,11 +151,15 @@ function signingClaimData(claim: ClaimObject): Buffer { * * @param transaction transaction to serialize * @param signingAccount Account to sign the transaction with + * @param opts.definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns serialized transaction with appropriate prefix and suffix */ function multiSigningData( transaction: JsonObject, signingAccount: string | AccountID, + opts: { definitions: XrplDefinitionsBase } = { + definitions: DEFAULT_DEFINITIONS, + }, ): Buffer { const prefix = HashPrefix.transactionMultiSig const suffix = coreTypes.AccountID.from(signingAccount).toBytes() @@ -135,6 +167,7 @@ function multiSigningData( prefix, suffix, signingFieldsOnly: true, + definitions: opts.definitions, }) } diff --git a/packages/ripple-binary-codec/src/coretypes.ts b/packages/ripple-binary-codec/src/coretypes.ts index 1a7f752a..d36ae7a9 100644 --- a/packages/ripple-binary-codec/src/coretypes.ts +++ b/packages/ripple-binary-codec/src/coretypes.ts @@ -1,4 +1,5 @@ import { + DEFAULT_DEFINITIONS, Field, TransactionType, LedgerEntryType, @@ -17,6 +18,7 @@ export { hashes, binary, ledgerHashes, + DEFAULT_DEFINITIONS, Field, TransactionType, LedgerEntryType, diff --git a/packages/ripple-binary-codec/src/enums/README.md b/packages/ripple-binary-codec/src/enums/README.md index 5f432260..bb127bdd 100644 --- a/packages/ripple-binary-codec/src/enums/README.md +++ b/packages/ripple-binary-codec/src/enums/README.md @@ -1,12 +1,16 @@ # Definitions +This file is used to serialize/deserialize transactions and ledger objects for the XRPL. It's broken into 5 sections laid out below. + +At the bottom of this README you can find instructions and examples for how to define your own types in a definitions file in order to work on a custom sidechain or develop new amendments. + ## Types These are the [types](https://xrpl.org/serialization.html#type-list) associated with a given Serialization Field. Each type has an arbitrary [type_code](https://xrpl.org/serialization.html#type-codes), with lower codes sorting first. ## Ledger Entry Types -Each ledger's state tree contain [ledger objects](https://xrpl.org/ledger-object-types.html), which represent all settings, balances, and relationships in the shared ledger. +Each ledger's state tree contain [ledger objects](https://xrpl.org/ledger-object-types.html), which represent all settings, balances, and relationships in the shared ledger. ## Fields @@ -53,8 +57,88 @@ See: - https://github.com/ripple/rippled/blob/develop/src/ripple/protocol/TER.h - https://xrpl.org/transaction-results.html -TODO: Write a script to read rippled's source file and generate the necessary mapping. +To generate a new definitions file from rippled source code, use this tool: https://github.com/RichardAH/xrpl-codec-gen ## Transaction Types See https://github.com/ripple/rippled/blob/develop/src/ripple/protocol/TxFormats.h + +# Defining Your Own Definitions + +If you're building your own sidechain or writing an amendment for the XRPL, you may need to create new XRPL definitions. + +To do that there are a couple things you need to do: + +1. Generate your own `definitions.json` file from rippled source code using [this tool](https://github.com/RichardAH/xrpl-codec-gen) (The default `definitions.json` for mainnet can be found [here](https://github.com/XRPLF/xrpl.js/blob/main/packages/ripple-binary-codec/src/enums/definitions.json)) +2. Create new SerializedType classes for any new Types (So that encode/decode behavior is defined). The SerializedType classes correspond to "ST..." classes in Rippled. Note: This is very rarely required. + +- For examples of how to implement that you can look at objects in the [`types` folder](../types/), such as `Amount`, `UInt8`, or `STArray`. + +3. Import your `definitions.json` file to construct your own `XrplDefinitions` object. +4. Pass the `XrplDefinitions` object whenever you `encode` or `decode` a transaction. +5. If you added any new transaction types, you should create an `interface` for the transaction that extends `BaseTransaction` from the `xrpl` repo to use it with the functions on `Client` (See the below example of adding a new transaction type) + +## Example of adding a new Transaction type + +``` +// newDefinitionsJson is where you can import your custom defined definitions.json file +const newDefinitionsJson = require('./new-transaction-type-definitions.json') +const { XrplDefinitions, Client } = require('xrpl') + +const newDefs = new XrplDefinitions(newDefinitionsJson) + +// Change to point at the server you care about +const serverAddress = 'wss://s.devnet.rippletest.net:51233' +const client = new Client(serverAddress) +const wallet1 = await client.fundWallet() + +// Extending BaseTransaction allows typescript to recognize this as a transaction type +interface NewTx extends BaseTransaction { + Amount: Amount +} + +const tx: NewTx = { + // The TransactionType here needs to match what you added in your newDefinitionsJson file + TransactionType: 'NewTx', + Account: wallet1.address, + Amount: '100', +} + +// By passing in your newDefs, your new transaction should be serializable. +// Rippled will still throw an error though if it's not a supported transaction type. +const result = await client.submitAndWait(tx, { + wallet: wallet1, + definitions: newDefs, +}) +``` + +## Example of adding a new serializable Type + +``` +const { XrplDefinitions } = require('../dist/coretypes') + +// newDefinitionsJson is where you can import your custom defined definitions.json file +const newDefinitionsJson = require('./fixtures/new-definitions.json') + + +// For any new Types you create, you'll need to make a class with the same name which extends a SerializedType object +// In order to define how to serialize/deserialize that field. Here we simply make our NewType act like a UInt32. + +const { UInt32 } = require('../dist/types/uint-32') +class NewType extends UInt32 { + // Should be the same as UInt32 +} + +const extendedCoreTypes = { NewType } + +const newDefs = new XrplDefinitions(newDefinitionsJson, extendedCoreTypes) + +// From this point on, we should be able to serialize / deserialize Transactions with fields that have 'NewType' as their Type. + +const encoded = encode(my_tx, newDefs) +const decoded = decode(encoded, newDefs) +``` + +## Other examples + +You can find other examples of how to modify `definitions.json` in `definition.test.js` which contains tests for this feature, and uses various example modified `definition` files. You can find the tests and the corresponding example `definition` files in [this folder of test cases](https://github.com/XRPLF/xrpl.js/tree/main/packages/ripple-binary-codec/test) diff --git a/packages/ripple-binary-codec/src/enums/bytes.ts b/packages/ripple-binary-codec/src/enums/bytes.ts new file mode 100644 index 00000000..9ef4b3c7 --- /dev/null +++ b/packages/ripple-binary-codec/src/enums/bytes.ts @@ -0,0 +1,75 @@ +import { BytesList, BinaryParser } from '../binary' +import { Buffer } from 'buffer/' + +/* + * @brief: Bytes, name, and ordinal representing one type, ledger_type, transaction type, or result + */ +export class Bytes { + readonly bytes: Buffer + + constructor( + readonly name: string, + readonly ordinal: number, + readonly ordinalWidth: number, + ) { + this.bytes = Buffer.alloc(ordinalWidth) + for (let i = 0; i < ordinalWidth; i++) { + this.bytes[ordinalWidth - i - 1] = (ordinal >>> (i * 8)) & 0xff + } + } + + toJSON(): string { + return this.name + } + + toBytesSink(sink: BytesList): void { + sink.put(this.bytes) + } + + toBytes(): Uint8Array { + return this.bytes + } +} + +/* + * @brief: Collection of Bytes objects, mapping bidirectionally + */ +export class BytesLookup { + constructor(types: Record, readonly ordinalWidth: number) { + Object.entries(types).forEach(([k, v]) => { + this.add(k, v) + }) + } + + /** + * Add a new name value pair to the BytesLookup. + * + * @param name - A human readable name for the field. + * @param value - The numeric value for the field. + * @throws if the name or value already exist in the lookup because it's unclear how to decode. + */ + add(name: string, value: number): void { + if (this[name]) { + throw new SyntaxError( + `Attempted to add a value with a duplicate name "${name}". This is not allowed because it is unclear how to decode.`, + ) + } + if (this[value.toString()]) { + throw new SyntaxError( + `Attempted to add a duplicate value under a different name (Given name: "${name}" and previous name: "${ + this[value.toString()] + }. This is not allowed because it is unclear how to decode.\nGiven value: ${value.toString()}`, + ) + } + this[name] = new Bytes(name, value, this.ordinalWidth) + this[value.toString()] = this[name] + } + + from(value: Bytes | string): Bytes { + return value instanceof Bytes ? value : (this[value] as Bytes) + } + + fromParser(parser: BinaryParser): Bytes { + return this.from(parser.readUIntN(this.ordinalWidth).toString()) + } +} diff --git a/packages/ripple-binary-codec/src/enums/constants.ts b/packages/ripple-binary-codec/src/enums/constants.ts new file mode 100644 index 00000000..7d181e47 --- /dev/null +++ b/packages/ripple-binary-codec/src/enums/constants.ts @@ -0,0 +1,4 @@ +export const TYPE_WIDTH = 2 +export const LEDGER_ENTRY_WIDTH = 2 +export const TRANSACTION_TYPE_WIDTH = 2 +export const TRANSACTION_RESULT_WIDTH = 1 diff --git a/packages/ripple-binary-codec/src/enums/field.ts b/packages/ripple-binary-codec/src/enums/field.ts new file mode 100644 index 00000000..31be02e0 --- /dev/null +++ b/packages/ripple-binary-codec/src/enums/field.ts @@ -0,0 +1,85 @@ +import { Bytes } from './bytes' +import { SerializedType } from '../types/serialized-type' +import { TYPE_WIDTH } from './constants' +import { Buffer } from 'buffer/' + +/** + * Encoding information for a rippled field, often used in transactions. + * See the enums [README.md](https://github.com/XRPLF/xrpl.js/tree/main/packages/ripple-binary-codec/src/enums) for more details on what each means. + */ +export interface FieldInfo { + nth: number + isVLEncoded: boolean + isSerialized: boolean + isSigningField: boolean + type: string +} + +export interface FieldInstance { + readonly nth: number + readonly isVariableLengthEncoded: boolean + readonly isSerialized: boolean + readonly isSigningField: boolean + readonly type: Bytes + readonly ordinal: number + readonly name: string + readonly header: Buffer + readonly associatedType: typeof SerializedType +} + +/* + * @brief: Serialize a field based on type_code and Field.nth + */ +function fieldHeader(type: number, nth: number): Buffer { + const header: Array = [] + if (type < 16) { + if (nth < 16) { + header.push((type << 4) | nth) + } else { + header.push(type << 4, nth) + } + } else if (nth < 16) { + header.push(nth, type) + } else { + header.push(0, type, nth) + } + return Buffer.from(header) +} + +function buildField( + [name, info]: [string, FieldInfo], + typeOrdinal: number, +): FieldInstance { + const field = fieldHeader(typeOrdinal, info.nth) + return { + name: name, + nth: info.nth, + isVariableLengthEncoded: info.isVLEncoded, + isSerialized: info.isSerialized, + isSigningField: info.isSigningField, + ordinal: (typeOrdinal << 16) | info.nth, + type: new Bytes(info.type, typeOrdinal, TYPE_WIDTH), + header: field, + associatedType: SerializedType, // For later assignment in ./types/index.js or Definitions.updateAll(...) + } +} + +/* + * @brief: The collection of all fields as defined in definitions.json + */ +export class FieldLookup { + constructor( + fields: Array<[string, FieldInfo]>, + types: Record, + ) { + fields.forEach(([name, field_info]) => { + const typeOrdinal = types[field_info.type] + this[name] = buildField([name, field_info], typeOrdinal) + this[this[name].ordinal.toString()] = this[name] + }) + } + + fromString(value: string): FieldInstance { + return this[value] as FieldInstance + } +} diff --git a/packages/ripple-binary-codec/src/enums/index.ts b/packages/ripple-binary-codec/src/enums/index.ts index ecfa54cc..965c0a98 100644 --- a/packages/ripple-binary-codec/src/enums/index.ts +++ b/packages/ripple-binary-codec/src/enums/index.ts @@ -1,164 +1,34 @@ import * as enums from './definitions.json' -import { SerializedType } from '../types/serialized-type' -import { Buffer } from 'buffer/' -import { BytesList } from '../binary' +import { + XrplDefinitionsBase, + FieldInstance, + Bytes, +} from './xrpl-definitions-base' +/** + * By default, coreTypes from the `types` folder is where known type definitions are initialized to avoid import cycles. + */ +const DEFAULT_DEFINITIONS = new XrplDefinitionsBase(enums, {}) + +const Type = DEFAULT_DEFINITIONS.type +const LedgerEntryType = DEFAULT_DEFINITIONS.ledgerEntryType +const TransactionType = DEFAULT_DEFINITIONS.transactionType +const TransactionResult = DEFAULT_DEFINITIONS.transactionResult +const Field = DEFAULT_DEFINITIONS.field /* * @brief: All valid transaction types */ -export const TRANSACTION_TYPES = Object.entries(enums.TRANSACTION_TYPES) - .filter(([_key, value]) => value >= 0) - .map(([key, _value]) => key) - -const TYPE_WIDTH = 2 -const LEDGER_ENTRY_WIDTH = 2 -const TRANSACTION_TYPE_WIDTH = 2 -const TRANSACTION_RESULT_WIDTH = 1 - -/* - * @brief: Serialize a field based on type_code and Field.nth - */ -function fieldHeader(type: number, nth: number): Buffer { - const header: Array = [] - if (type < 16) { - if (nth < 16) { - header.push((type << 4) | nth) - } else { - header.push(type << 4, nth) - } - } else if (nth < 16) { - header.push(nth, type) - } else { - header.push(0, type, nth) - } - return Buffer.from(header) -} - -/* - * @brief: Bytes, name, and ordinal representing one type, ledger_type, transaction type, or result - */ -export class Bytes { - readonly bytes: Buffer - - constructor( - readonly name: string, - readonly ordinal: number, - readonly ordinalWidth: number, - ) { - this.bytes = Buffer.alloc(ordinalWidth) - for (let i = 0; i < ordinalWidth; i++) { - this.bytes[ordinalWidth - i - 1] = (ordinal >>> (i * 8)) & 0xff - } - } - - toJSON(): string { - return this.name - } - - toBytesSink(sink: BytesList): void { - sink.put(this.bytes) - } - - toBytes(): Uint8Array { - return this.bytes - } -} - -/* - * @brief: Collection of Bytes objects, mapping bidirectionally - */ -class BytesLookup { - constructor(types: Record, readonly ordinalWidth: number) { - Object.entries(types).forEach(([k, v]) => { - this[k] = new Bytes(k, v, ordinalWidth) - this[v.toString()] = this[k] - }) - } - - from(value: Bytes | string): Bytes { - return value instanceof Bytes ? value : (this[value] as Bytes) - } - - fromParser(parser): Bytes { - return this.from(parser.readUIntN(this.ordinalWidth).toString()) - } -} - -/* - * type FieldInfo is the type of the objects containing information about each field in definitions.json - */ -interface FieldInfo { - nth: number - isVLEncoded: boolean - isSerialized: boolean - isSigningField: boolean - type: string -} - -interface FieldInstance { - readonly nth: number - readonly isVariableLengthEncoded: boolean - readonly isSerialized: boolean - readonly isSigningField: boolean - readonly type: Bytes - readonly ordinal: number - readonly name: string - readonly header: Buffer - readonly associatedType: typeof SerializedType -} - -function buildField([name, info]: [string, FieldInfo]): FieldInstance { - const typeOrdinal = enums.TYPES[info.type] - const field = fieldHeader(typeOrdinal, info.nth) - return { - name: name, - nth: info.nth, - isVariableLengthEncoded: info.isVLEncoded, - isSerialized: info.isSerialized, - isSigningField: info.isSigningField, - ordinal: (typeOrdinal << 16) | info.nth, - type: new Bytes(info.type, typeOrdinal, TYPE_WIDTH), - header: field, - associatedType: SerializedType, // For later assignment in ./types/index.js - } -} - -/* - * @brief: The collection of all fields as defined in definitions.json - */ -class FieldLookup { - constructor(fields: Array<[string, FieldInfo]>) { - fields.forEach(([k, v]) => { - this[k] = buildField([k, v]) - this[this[k].ordinal.toString()] = this[k] - }) - } - - fromString(value: string): FieldInstance { - return this[value] as FieldInstance - } -} - -const Type = new BytesLookup(enums.TYPES, TYPE_WIDTH) -const LedgerEntryType = new BytesLookup( - enums.LEDGER_ENTRY_TYPES, - LEDGER_ENTRY_WIDTH, -) -const TransactionType = new BytesLookup( - enums.TRANSACTION_TYPES, - TRANSACTION_TYPE_WIDTH, -) -const TransactionResult = new BytesLookup( - enums.TRANSACTION_RESULTS, - TRANSACTION_RESULT_WIDTH, -) -const Field = new FieldLookup(enums.FIELDS as Array<[string, FieldInfo]>) +const TRANSACTION_TYPES = DEFAULT_DEFINITIONS.transactionNames export { + Bytes, + XrplDefinitionsBase, + DEFAULT_DEFINITIONS, Field, FieldInstance, Type, LedgerEntryType, TransactionResult, TransactionType, + TRANSACTION_TYPES, } diff --git a/packages/ripple-binary-codec/src/enums/xrpl-definitions-base.ts b/packages/ripple-binary-codec/src/enums/xrpl-definitions-base.ts new file mode 100644 index 00000000..1a61a314 --- /dev/null +++ b/packages/ripple-binary-codec/src/enums/xrpl-definitions-base.ts @@ -0,0 +1,111 @@ +import { SerializedType } from '../types/serialized-type' +import { Bytes, BytesLookup } from './bytes' +import { FieldInfo, FieldLookup, FieldInstance } from './field' +import { + TYPE_WIDTH, + LEDGER_ENTRY_WIDTH, + TRANSACTION_TYPE_WIDTH, + TRANSACTION_RESULT_WIDTH, +} from './constants' + +interface DefinitionsData { + TYPES: Record + LEDGER_ENTRY_TYPES: Record + FIELDS: (string | FieldInfo)[][] + TRANSACTION_RESULTS: Record + TRANSACTION_TYPES: Record +} + +/** + * Stores the various types and fields for rippled to be used to encode/decode information later on. + * XrplDefinitions should be instantiated instead of this class. + */ +class XrplDefinitionsBase { + // A collection of fields that can be included in transactions + field: FieldLookup + // A collection of ids corresponding to types of ledger objects + ledgerEntryType: BytesLookup + // A collection of type flags used to determine how to serialize a field's data + type: BytesLookup + // Errors and result codes for transactions + transactionResult: BytesLookup + // Defined transactions that can be submitted to the ledger + transactionType: BytesLookup + // Valid transaction names + transactionNames: string[] + // Maps serializable types to their TypeScript class implementation + dataTypes: Record + + /** + * Present rippled types in a typed and updatable format. + * For an example of the input format see `definitions.json` + * To generate a new definitions file from rippled source code, use this tool: https://github.com/RichardAH/xrpl-codec-gen + * + * See the definitions.test.js file for examples of how to create your own updated definitions.json. + * + * @param enums - A json encoding of the core types, transaction types, transaction results, transaction names, and fields. + * @param types - A list of type objects with the same name as the fields defined. + * You can use the coreTypes object if you are not adding new types. + */ + constructor( + enums: DefinitionsData, + types: Record, + ) { + this.type = new BytesLookup(enums.TYPES, TYPE_WIDTH) + this.ledgerEntryType = new BytesLookup( + enums.LEDGER_ENTRY_TYPES, + LEDGER_ENTRY_WIDTH, + ) + this.transactionType = new BytesLookup( + enums.TRANSACTION_TYPES, + TRANSACTION_TYPE_WIDTH, + ) + this.transactionResult = new BytesLookup( + enums.TRANSACTION_RESULTS, + TRANSACTION_RESULT_WIDTH, + ) + this.field = new FieldLookup( + enums.FIELDS as Array<[string, FieldInfo]>, + enums.TYPES, + ) + this.transactionNames = Object.entries(enums.TRANSACTION_TYPES) + .filter(([_key, value]) => value >= 0) + .map(([key, _value]) => key) + + this.dataTypes = {} // Filled in via associateTypes + this.associateTypes(types) + } + + /** + * Associates each Field to a corresponding class that TypeScript can recognize. + * + * @param types a list of type objects with the same name as the fields defined. + * Defaults to xrpl.js's core type definitions. + */ + public associateTypes(types: Record): void { + // Overwrite any existing type definitions with the given types + this.dataTypes = Object.assign({}, this.dataTypes, types) + + Object.values(this.field).forEach((field) => { + field.associatedType = this.dataTypes[field.type.name] + }) + + this.field['TransactionType'].associatedType = this.transactionType + this.field['TransactionResult'].associatedType = this.transactionResult + this.field['LedgerEntryType'].associatedType = this.ledgerEntryType + } + + public getAssociatedTypes(): Record { + return this.dataTypes + } +} + +export { + DefinitionsData, + XrplDefinitionsBase, + FieldLookup, + FieldInfo, + FieldInstance, + Bytes, + BytesLookup, +} diff --git a/packages/ripple-binary-codec/src/enums/xrpl-definitions.ts b/packages/ripple-binary-codec/src/enums/xrpl-definitions.ts new file mode 100644 index 00000000..30f8ec90 --- /dev/null +++ b/packages/ripple-binary-codec/src/enums/xrpl-definitions.ts @@ -0,0 +1,32 @@ +import { + type DefinitionsData, + XrplDefinitionsBase, +} from './xrpl-definitions-base' +import { coreTypes } from '../types' +import { SerializedType } from '../types/serialized-type' + +/** + * Stores the various types and fields for rippled to be used to encode/decode information later on. + * Should be used instead of XrplDefinitionsBase since this defines default `types` for serializing/deserializing + * ledger data. + */ +export class XrplDefinitions extends XrplDefinitionsBase { + /** + * Present rippled types in a typed and updatable format. + * For an example of the input format see `definitions.json` + * To generate a new definitions file from rippled source code, use this tool: https://github.com/RichardAH/xrpl-codec-gen + * + * See the definitions.test.js file for examples of how to create your own updated definitions.json. + * + * @param enums - A json encoding of the core types, transaction types, transaction results, transaction names, and fields. + * @param additionalTypes - A list of SerializedType objects with the same name as the fields defined. + * These types will be included in addition to the coreTypes used on mainnet. + */ + constructor( + enums: DefinitionsData, + additionalTypes?: Record, + ) { + const types = Object.assign({}, coreTypes, additionalTypes) + super(enums, types) + } +} diff --git a/packages/ripple-binary-codec/src/index.ts b/packages/ripple-binary-codec/src/index.ts index 6ee0f584..6d7852d2 100644 --- a/packages/ripple-binary-codec/src/index.ts +++ b/packages/ripple-binary-codec/src/index.ts @@ -1,9 +1,15 @@ import * as assert from 'assert' -import { quality, binary } from './coretypes' +import { quality, binary, HashPrefix } from './coretypes' import { decodeLedgerData } from './ledger-hashes' import { ClaimObject } from './binary' import { JsonObject } from './types/serialized-type' -import { TRANSACTION_TYPES } from './enums' +import { + XrplDefinitionsBase, + TRANSACTION_TYPES, + DEFAULT_DEFINITIONS, +} from './enums' +import { XrplDefinitions } from './enums/xrpl-definitions' +import { coreTypes } from './types' const { signingData, @@ -17,22 +23,25 @@ const { * Decode a transaction * * @param binary hex-string of the encoded transaction + * @param definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns the JSON representation of the transaction */ -function decode(binary: string): JsonObject { +function decode(binary: string, definitions?: XrplDefinitionsBase): JsonObject { assert.ok(typeof binary === 'string', 'binary must be a hex string') - return binaryToJSON(binary) + return binaryToJSON(binary, definitions) } /** * Encode a transaction * * @param json The JSON representation of a transaction + * @param definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. + * * @returns A hex-string of the encoded transaction */ -function encode(json: object): string { +function encode(json: object, definitions?: XrplDefinitionsBase): string { assert.ok(typeof json === 'object') - return serializeObject(json as JsonObject) + return serializeObject(json as JsonObject, { definitions }) .toString('hex') .toUpperCase() } @@ -42,11 +51,17 @@ function encode(json: object): string { * * @param json JSON object representing the transaction * @param signer string representing the account to sign the transaction with + * @param definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns a hex string of the encoded transaction */ -function encodeForSigning(json: object): string { +function encodeForSigning( + json: object, + definitions?: XrplDefinitionsBase, +): string { assert.ok(typeof json === 'object') - return signingData(json as JsonObject) + return signingData(json as JsonObject, HashPrefix.transactionSig, { + definitions, + }) .toString('hex') .toUpperCase() } @@ -56,6 +71,7 @@ function encodeForSigning(json: object): string { * * @param json JSON object representing the transaction * @param signer string representing the account to sign the transaction with + * @param definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns a hex string of the encoded transaction */ function encodeForSigningClaim(json: object): string { @@ -70,12 +86,18 @@ function encodeForSigningClaim(json: object): string { * * @param json JSON object representing the transaction * @param signer string representing the account to sign the transaction with + * @param definitions Custom rippled types to use instead of the default. Used for sidechains and amendments. * @returns a hex string of the encoded transaction */ -function encodeForMultisigning(json: object, signer: string): string { +function encodeForMultisigning( + json: object, + signer: string, + definitions?: XrplDefinitionsBase, +): string { assert.ok(typeof json === 'object') assert.equal(json['SigningPubKey'], '') - return multiSigningData(json as JsonObject, signer) + const definitionsOpt = definitions ? { definitions } : undefined + return multiSigningData(json as JsonObject, signer, definitionsOpt) .toString('hex') .toUpperCase() } @@ -112,4 +134,8 @@ export { decodeQuality, decodeLedgerData, TRANSACTION_TYPES, + XrplDefinitions, + XrplDefinitionsBase, + DEFAULT_DEFINITIONS, + coreTypes, } diff --git a/packages/ripple-binary-codec/src/ledger-hashes.ts b/packages/ripple-binary-codec/src/ledger-hashes.ts index d88b07ce..92f5b03f 100644 --- a/packages/ripple-binary-codec/src/ledger-hashes.ts +++ b/packages/ripple-binary-codec/src/ledger-hashes.ts @@ -11,6 +11,7 @@ import { UInt8 } from './types/uint-8' import { BinaryParser } from './serdes/binary-parser' import { JsonObject } from './types/serialized-type' import bigInt = require('big-integer') +import { XrplDefinitionsBase } from './enums' /** * Computes the hash of a list of objects @@ -160,11 +161,16 @@ function ledgerHash(header: ledgerObject): Hash256 { * Decodes a serialized ledger header * * @param binary A serialized ledger header + * @param definitions Type definitions to parse the ledger objects. + * Used if there are non-default ledger objects to decode. * @returns A JSON object describing a ledger header */ -function decodeLedgerData(binary: string): object { +function decodeLedgerData( + binary: string, + definitions?: XrplDefinitionsBase, +): object { assert.ok(typeof binary === 'string', 'binary must be a hex string') - const parser = new BinaryParser(binary) + const parser = new BinaryParser(binary, definitions) return { ledger_index: parser.readUInt32(), total_coins: parser.readType(UInt64).valueOf().toString(), diff --git a/packages/ripple-binary-codec/src/serdes/binary-parser.ts b/packages/ripple-binary-codec/src/serdes/binary-parser.ts index 534ef52f..bd654580 100644 --- a/packages/ripple-binary-codec/src/serdes/binary-parser.ts +++ b/packages/ripple-binary-codec/src/serdes/binary-parser.ts @@ -1,6 +1,10 @@ import * as assert from 'assert' -import { Field, FieldInstance } from '../enums' -import { SerializedType } from '../types/serialized-type' +import { + XrplDefinitionsBase, + DEFAULT_DEFINITIONS, + FieldInstance, +} from '../enums' +import { type SerializedType } from '../types/serialized-type' import { Buffer } from 'buffer/' /** @@ -8,14 +12,21 @@ import { Buffer } from 'buffer/' */ class BinaryParser { private bytes: Buffer + definitions: XrplDefinitionsBase /** * Initialize bytes to a hex string * * @param hexBytes a hex string + * @param definitions Rippled definitions used to parse the values of transaction types and such. + * Can be customized for sidechains and amendments. */ - constructor(hexBytes: string) { + constructor( + hexBytes: string, + definitions: XrplDefinitionsBase = DEFAULT_DEFINITIONS, + ) { this.bytes = Buffer.from(hexBytes, 'hex') + this.definitions = definitions } /** @@ -146,7 +157,7 @@ class BinaryParser { * @return The field represented by the bytes at the head of the BinaryParser */ readField(): FieldInstance { - return Field.fromString(this.readFieldOrdinal().toString()) + return this.definitions.field.fromString(this.readFieldOrdinal().toString()) } /** diff --git a/packages/ripple-binary-codec/src/serdes/binary-serializer.ts b/packages/ripple-binary-codec/src/serdes/binary-serializer.ts index b8ff193b..bf060211 100644 --- a/packages/ripple-binary-codec/src/serdes/binary-serializer.ts +++ b/packages/ripple-binary-codec/src/serdes/binary-serializer.ts @@ -1,6 +1,6 @@ import * as assert from 'assert' import { FieldInstance } from '../enums' -import { SerializedType } from '../types/serialized-type' +import { type SerializedType } from '../types/serialized-type' import { Buffer } from 'buffer/' /** diff --git a/packages/ripple-binary-codec/src/shamap.ts b/packages/ripple-binary-codec/src/shamap.ts index c44fd12b..a2b26bf1 100644 --- a/packages/ripple-binary-codec/src/shamap.ts +++ b/packages/ripple-binary-codec/src/shamap.ts @@ -130,7 +130,7 @@ class ShaMapInner extends ShaMapNode { */ hash(): Hash256 { if (this.empty()) { - return coreTypes.Hash256.ZERO_256 + return (coreTypes.Hash256 as typeof Hash256).ZERO_256 } const hash = Sha512Half.put(this.hashPrefix()) this.toBytesSink(hash) @@ -145,7 +145,9 @@ class ShaMapInner extends ShaMapNode { toBytesSink(list: BytesList): void { for (let i = 0; i < this.branches.length; i++) { const branch = this.branches[i] - const hash = branch ? branch.hash() : coreTypes.Hash256.ZERO_256 + const hash = branch + ? branch.hash() + : (coreTypes.Hash256 as typeof Hash256).ZERO_256 hash.toBytesSink(list) } } diff --git a/packages/ripple-binary-codec/src/types/index.ts b/packages/ripple-binary-codec/src/types/index.ts index 5efa199a..fd8b02c2 100644 --- a/packages/ripple-binary-codec/src/types/index.ts +++ b/packages/ripple-binary-codec/src/types/index.ts @@ -1,9 +1,3 @@ -import { - Field, - TransactionResult, - TransactionType, - LedgerEntryType, -} from '../enums' import { AccountID } from './account-id' import { Amount } from './amount' import { Blob } from './blob' @@ -19,8 +13,10 @@ import { UInt32 } from './uint-32' import { UInt64 } from './uint-64' import { UInt8 } from './uint-8' import { Vector256 } from './vector-256' +import { type SerializedType } from './serialized-type' +import { DEFAULT_DEFINITIONS } from '../enums' -const coreTypes = { +const coreTypes: Record = { AccountID, Amount, Blob, @@ -38,12 +34,26 @@ const coreTypes = { Vector256, } -Object.values(Field).forEach((field) => { - field.associatedType = coreTypes[field.type.name] -}) +// Ensures that the DEFAULT_DEFINITIONS object connects these types to fields for serializing/deserializing +// This is done here instead of in enums/index.ts to avoid a circular dependency +// because some of the above types depend on BinarySerailizer which depends on enums/index.ts. +DEFAULT_DEFINITIONS.associateTypes(coreTypes) -Field['TransactionType'].associatedType = TransactionType -Field['TransactionResult'].associatedType = TransactionResult -Field['LedgerEntryType'].associatedType = LedgerEntryType - -export { coreTypes } +export { + coreTypes, + AccountID, + Amount, + Blob, + Currency, + Hash128, + Hash160, + Hash256, + PathSet, + STArray, + STObject, + UInt8, + UInt16, + UInt32, + UInt64, + Vector256, +} diff --git a/packages/ripple-binary-codec/src/types/st-object.ts b/packages/ripple-binary-codec/src/types/st-object.ts index 519dda52..8cd631fc 100644 --- a/packages/ripple-binary-codec/src/types/st-object.ts +++ b/packages/ripple-binary-codec/src/types/st-object.ts @@ -1,4 +1,9 @@ -import { Field, FieldInstance, Bytes } from '../enums' +import { + DEFAULT_DEFINITIONS, + FieldInstance, + Bytes, + XrplDefinitionsBase, +} from '../enums' import { SerializedType, JsonObject } from './serialized-type' import { xAddressToClassicAddress, isValidXAddress } from 'ripple-address-codec' import { BinaryParser } from '../serdes/binary-parser' @@ -83,11 +88,13 @@ class STObject extends SerializedType { * * @param value An object to include * @param filter optional, denote which field to include in serialized object + * @param definitions optional, types and values to use to encode/decode a transaction * @returns a STObject object */ static from( value: T, filter?: (...any) => boolean, + definitions: XrplDefinitionsBase = DEFAULT_DEFINITIONS, ): STObject { if (value instanceof STObject) { return value @@ -108,7 +115,7 @@ class STObject extends SerializedType { }, {}) let sorted = Object.keys(xAddressDecoded) - .map((f: string): FieldInstance => Field[f] as FieldInstance) + .map((f: string): FieldInstance => definitions.field[f] as FieldInstance) .filter( (f: FieldInstance): boolean => f !== undefined && @@ -155,11 +162,12 @@ class STObject extends SerializedType { /** * Get the JSON interpretation of this.bytes - * + * @param definitions rippled definitions used to parse the values of transaction types and such. + * Can be customized for sidechains and amendments. * @returns a JSON object */ - toJSON(): JsonObject { - const objectParser = new BinaryParser(this.toString()) + toJSON(definitions?: XrplDefinitionsBase): JsonObject { + const objectParser = new BinaryParser(this.toString(), definitions) const accumulator = {} while (!objectParser.end()) { diff --git a/packages/ripple-binary-codec/test/definitions.test.js b/packages/ripple-binary-codec/test/definitions.test.js new file mode 100644 index 00000000..3ac575b3 --- /dev/null +++ b/packages/ripple-binary-codec/test/definitions.test.js @@ -0,0 +1,100 @@ +const { encode, decode, XrplDefinitions } = require('../src') +const normalDefinitionsJson = require('../src/enums/definitions.json') +const { UInt32 } = require('../dist/types/uint-32') + +const txJson = { + Account: 'r9LqNeG6qHxjeUocjvVki2XR35weJ9mZgQ', + Amount: '1000', + Destination: 'rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh', + Fee: '10', + Flags: 0, + Sequence: 1, + TransactionType: 'Payment', +} + +describe('encode and decode using new types as a parameter', function () { + test('can encode and decode a new TransactionType', function () { + const tx = { ...txJson, TransactionType: 'NewTestTransaction' } + // Before updating the types, this should not be encodable + expect(() => encode(tx)).toThrow() + + // Normally this would be generated directly from rippled with something like `server_definitions`. + // Added here to make it easier to see what is actually changing in the definitions.json file. + const definitions = JSON.parse(JSON.stringify(normalDefinitionsJson)) + definitions.TRANSACTION_TYPES['NewTestTransaction'] = 30 + + const newDefs = new XrplDefinitions(definitions) + + const encoded = encode(tx, newDefs) + expect(() => decode(encoded)).toThrow() + const decoded = decode(encoded, newDefs) + expect(decoded).toStrictEqual(tx) + }) + + test('can encode and decode a new Field', function () { + const tx = { ...txJson, NewFieldDefinition: 10 } + + // Before updating the types, undefined fields will be ignored on encode + expect(decode(encode(tx))).not.toStrictEqual(tx) + + // Normally this would be generated directly from rippled with something like `server_definitions`. + // Added here to make it easier to see what is actually changing in the definitions.json file. + const definitions = JSON.parse(JSON.stringify(normalDefinitionsJson)) + + definitions.FIELDS.push([ + 'NewFieldDefinition', + { + nth: 100, + isVLEncoded: false, + isSerialized: true, + isSigningField: true, + type: 'UInt32', + }, + ]) + + const newDefs = new XrplDefinitions(definitions) + + const encoded = encode(tx, newDefs) + expect(() => decode(encoded)).toThrow() + const decoded = decode(encoded, newDefs) + expect(decoded).toStrictEqual(tx) + }) + + test('can encode and decode a new Type', function () { + const tx = { + ...txJson, + TestField: 10, // Should work the same as a UInt32 + } + + // Normally this would be generated directly from rippled with something like `server_definitions`. + // Added here to make it easier to see what is actually changing in the definitions.json file. + const definitions = JSON.parse(JSON.stringify(normalDefinitionsJson)) + definitions.TYPES.NewType = 24 + definitions.FIELDS.push([ + 'TestField', + { + nth: 100, + isVLEncoded: true, + isSerialized: true, + isSigningField: true, + type: 'NewType', + }, + ]) + + // Test that before updating the types this tx fails to decode correctly. Note that undefined fields are ignored on encode. + expect(decode(encode(tx))).not.toStrictEqual(tx) + + class NewType extends UInt32 { + // Should be the same as UInt32 + } + + const extendedCoreTypes = { NewType } + + const newDefs = new XrplDefinitions(definitions, extendedCoreTypes) + + const encoded = encode(tx, newDefs) + expect(() => decode(encoded)).toThrow() + const decoded = decode(encoded, newDefs) + expect(decoded).toStrictEqual(tx) + }) +}) diff --git a/packages/ripple-binary-codec/test/signing-data-encoding.test.js b/packages/ripple-binary-codec/test/signing-data-encoding.test.js index 18d410c4..71a0c26c 100644 --- a/packages/ripple-binary-codec/test/signing-data-encoding.test.js +++ b/packages/ripple-binary-codec/test/signing-data-encoding.test.js @@ -4,6 +4,9 @@ const { encodeForSigningClaim, encodeForMultisigning, } = require('../src') +const { XrplDefinitions } = require('../src/enums/xrpl-definitions') + +const normalDefinitions = require('../src/enums/definitions.json') const tx_json = { Account: 'r9LqNeG6qHxjeUocjvVki2XR35weJ9mZgQ', @@ -67,6 +70,53 @@ describe('Signing data', function () { ) }) + test('can create single signing blobs with modified type', function () { + const customPaymentDefinitions = JSON.parse( + JSON.stringify(normalDefinitions), + ) + customPaymentDefinitions.TRANSACTION_TYPES.Payment = 31 + + const newDefs = new XrplDefinitions(customPaymentDefinitions) + const actual = encodeForSigning(tx_json, newDefs) + expect(actual).toBe( + [ + '53545800', // signingPrefix + // TransactionType + '12', + '001F', + // Flags + '22', + '80000000', + // Sequence + '24', + '00000001', + // Amount + '61', + // native amount + '40000000000003E8', + // Fee + '68', + // native amount + '400000000000000A', + // SigningPubKey + '73', + // VLLength + '21', + 'ED5F5AC8B98974A3CA843326D9B88CEBD0560177B973EE0B149F782CFAA06DC66A', + // Account + '81', + // VLLength + '14', + '5B812C9D57731E27A2DA8B1830195F88EF32A3B6', + // Destination + '83', + // VLLength + '14', + 'B5F762798A53D543A014CAF8B297CFF8F2F937E8', + ].join(''), + ) + }) + test('can fail gracefully for invalid TransactionType', function () { const invalidTransactionType = { ...tx_json, @@ -78,7 +128,7 @@ describe('Signing data', function () { test('can create multi signing blobs', function () { const signingAccount = 'rJZdUusLDtY9NEsGea7ijqhVrXv98rYBYN' - const signingJson = Object.assign({}, tx_json, { SigningPubKey: '' }) + const signingJson = { ...tx_json, SigningPubKey: '' } const actual = encodeForMultisigning(signingJson, signingAccount) expect(actual).toBe( [ @@ -120,6 +170,58 @@ describe('Signing data', function () { ].join(''), ) }) + + test('can create multi signing blobs with custom definitions', function () { + const customPaymentDefinitions = JSON.parse( + JSON.stringify(normalDefinitions), + ) + customPaymentDefinitions.TRANSACTION_TYPES.Payment = 31 + + const newDefs = new XrplDefinitions(customPaymentDefinitions) + const signingAccount = 'rJZdUusLDtY9NEsGea7ijqhVrXv98rYBYN' + const signingJson = { ...tx_json, SigningPubKey: '' } + const actual = encodeForMultisigning(signingJson, signingAccount, newDefs) + expect(actual).toBe( + [ + '534D5400', // signingPrefix + // TransactionType + '12', + '001F', + // Flags + '22', + '80000000', + // Sequence + '24', + '00000001', + // Amount + '61', + // native amount + '40000000000003E8', + // Fee + '68', + // native amount + '400000000000000A', + // SigningPubKey + '73', + // VLLength + '00', + // '', + // Account + '81', + // VLLength + '14', + '5B812C9D57731E27A2DA8B1830195F88EF32A3B6', + // Destination + '83', + // VLLength + '14', + 'B5F762798A53D543A014CAF8B297CFF8F2F937E8', + // signingAccount suffix + 'C0A5ABEF242802EFED4B041E8F2D4A8CC86AE3D1', + ].join(''), + ) + }) + test('can create claim blob', function () { const channel = '43904CBFCDCEC530B4037871F86EE90BF799DF8D2E0EA564BC8A3F332E4F5FB1' diff --git a/packages/xrpl/test/wallet/signer.test.ts b/packages/xrpl/test/wallet/signer.test.ts index de4a963f..108c89e2 100644 --- a/packages/xrpl/test/wallet/signer.test.ts +++ b/packages/xrpl/test/wallet/signer.test.ts @@ -139,7 +139,9 @@ describe('Signer', function () { it('multisign runs successfully with tx_blobs', function () { const transactions = [multisignTxToCombine1, multisignTxToCombine2] - const encodedTransactions: string[] = transactions.map(encode) + const encodedTransactions: string[] = transactions.map((transaction) => + encode(transaction), + ) assert.deepEqual(multisign(encodedTransactions), expectedMultisign) })