diff --git a/content/_code-samples/tx-serialization/serialize.py b/content/_code-samples/tx-serialization/serialize.py index c76a32c227..869aa3fc46 100755 --- a/content/_code-samples/tx-serialization/serialize.py +++ b/content/_code-samples/tx-serialization/serialize.py @@ -6,6 +6,7 @@ import json import logging +import re from address import decode_address @@ -43,12 +44,31 @@ def field_id(field_name): field_code = DEFINITIONS["FIELDS"][field_name]["nth"] if type_code < 16 and field_code < 16: - # first 4 bits is the type_code, next 4 bits is the field code + # high 4 bits is the type_code + # low 4 bits is the field code combined_code = (type_code << 4) | field_code - return combined_code.to_bytes(1, byteorder="big", signed=False) - else: - # TODO: need more than 1 byte to encode this field id - raise ValueError("field_id not yet implemented for types/fields > 16") + return bytes_from_uint(combined_code, 8) + elif type_code >= 16 and field_code < 16: + # first 4 bits are zeroes + # next 4 bits is field code + # next byte is type code + byte1 = bytes_from_uint(field_code, 8) + byte2 = bytes_from_uint(type_code, 8) + return b''.join( (byte1, byte2) ) + elif type_code < 16 and field_code >= 16: + # first 4 bits is type code + # next 4 bits are zeroes + # next byte is field code + byte1 = bytes_from_uint(type_code << 4, 8) + byte2 = bytes_from_uint(field_code, 8) + return b''.join( (byte1, byte2) ) + else: # both are >= 16 + # first byte is all zeroes + # second byte is type + # third byte is field code + byte2 = bytes_from_uint(type_code, 8) + byte3 = bytes_from_uint(field_code, 8) + return b''.join( (bytes(1), byte2, byte3) ) def bytes_from_uint(i, bits): if bits % 8: @@ -74,8 +94,7 @@ def amount_to_bytes(a): # https://developers.ripple.com/currency-formats.html#issued-currency-math # temporarily returning all zeroes issued_amt = bytes(8) - # TODO: calculate 160-bit currency ID - currency_code = bytes(20) + currency_code = currency_code_to_bytes(a["currency"]) return issued_amt + currency_code + decode_address(a["issuer"]) else: raise ValueError("amount must be XRP string or {currency, value, issuer}") @@ -84,6 +103,51 @@ def tx_type_to_bytes(txtype): type_uint = DEFINITIONS["TRANSACTION_TYPES"][txtype] return type_uint.to_bytes(2, byteorder="big", signed=False) +def currency_code_to_bytes(code_string): + if re.match(r"^[A-Za-z0-9?!@#$%^&*<>(){}\[\]|]{3}$", code_string): + # ISO 4217-like code + if code_string == "XRP": + raise ValueError("issued currency can't be XRP") + code_ascii = code_string.encode("ASCII") + # standard currency codes: https://developers.ripple.com/currency-formats.html#standard-currency-codes + # 8 bits type code (0x00) + # 96 bits reserved (0's) + # 24 bits ASCII + # 8 bits version (0x00) + # 24 bits reserved (0's) + return b''.join( ( bytes(13), code_ascii, bytes(4) ) ) + elif re.match(r"^[0-9a-fA-F]{40}$", code_string): + # raw hex code + return bytes.fromhex(code_string) # requires Python 3.5+ + else: + raise ValueError("invalid currency code") + +def vl_encode(vl_contents): + vl_len = len(vl_contents) + if vl_len <= 192: + len_byte = vl_len.to_bytes(1, byteorder="big", signed=False) + return b''.join( (len_byte, vl_contents) ) + elif vl_len <= 12480: + vl_len -= 193 + byte1 = ((vl_len >> 8) + 193).to_bytes(1, byteorder="big", signed=False) + byte2 = (vl_len & 0xff).to_bytes(1, byteorder="big", signed=False) + return b''.join( (byte1, byte2, vl_contents) ) + elif vl_len <= 918744: + vl_len -= 12481 + byte1 = (241 + (vl_len >> 16)).to_bytes(1, byteorder="big", signed=False) + byte2 = ((vl_len >> 8) & 0xff).to_bytes(1, byteorder="big", signed=False) + byte3 = (vl_len & 0xff).to_bytes(1, byteorder="big", signed=False) + return b''.join( (byte1, byte2, byte3, vl_contents) ) + + raise ValueError("VariableLength field must be <= 918744 bytes long") + +def vl_to_bytes(field_val): + vl_contents = bytes.fromhex(field_val) + return vl_encode(vl_contents) + +def accountid_to_bytes(address): + return vl_encode(decode_address(address)) + def field_to_bytes(field_name, field_val): field_type = DEFINITIONS["FIELDS"][field_name]["type"] logger.debug("Serializing field {f} of type {t}".format(f=field_name, t=field_type)) @@ -101,8 +165,9 @@ def field_to_bytes(field_name, field_val): "UInt32": lambda x:bytes_from_uint(x, 32), "UInt16": lambda x:bytes_from_uint(x, 16), "UInt8" : lambda x:bytes_from_uint(x, 8), - "AccountID": decode_address, - "Amount": amount_to_bytes + "AccountID": accountid_to_bytes, + "Amount": amount_to_bytes, + "Blob": vl_to_bytes } field_binary = dispatch[field_type](field_val) return b''.join( (id_prefix, field_binary) ) @@ -113,8 +178,11 @@ def serialize_tx(tx): fields_as_bytes = [] for field_name in field_order: - field_val = tx[field_name] - fields_as_bytes.append(field_to_bytes(field_name, field_val)) + if (DEFINITIONS["FIELDS"][field_name]["isSerialized"]): + field_val = tx[field_name] + field_bytes = field_to_bytes(field_name, field_val) + logger.debug("{n}: {h}".format(n=field_name, h=field_bytes.hex())) + fields_as_bytes.append(field_bytes) all_serial = b''.join(fields_as_bytes) logger.info(all_serial.hex().upper()) @@ -128,19 +196,22 @@ if __name__ == "__main__": logger.setLevel(logging.DEBUG) DEFINITIONS = load_defs() - example_tx = { - "TransactionType" : "Payment", - "Account" : "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", - "Destination" : "ra5nK24KXen9AHvsdFTKHSANinZseWnPcX", - "Amount" : { - "currency" : "USD", - "value" : "1", - "issuer" : "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn" - }, - "Fee": "12", - "Flags": 2147483648, - "Sequence": 2 - } + # example_tx = { + # "TransactionType" : "Payment", + # "Account" : "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh", + # "Destination" : "ra5nK24KXen9AHvsdFTKHSANinZseWnPcX", + # "Amount" : { + # "currency" : "USD", + # "value" : "1", + # "issuer" : "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn" + # }, + # "Fee": "12", + # "Flags": 2147483648, + # "Sequence": 2 + # } + + with open("test-cases/tx1-nometa.json") as f: + example_tx = json.load(f) serialize_tx(example_tx) diff --git a/content/_code-samples/tx-serialization/test-cases/tx1-binary.txt b/content/_code-samples/tx-serialization/test-cases/tx1-binary.txt new file mode 100644 index 0000000000..4b66a67703 --- /dev/null +++ b/content/_code-samples/tx-serialization/test-cases/tx1-binary.txt @@ -0,0 +1 @@ +120007220008000024001ABED82A2380BF2C2019001ABED764D55920AC9391400000000000000000000000000055534400000000000A20B3C85F482532A9578DBB3950B85CA06594D165400000037E11D60068400000000000000A732103EE83BB432547885C219634A1BC407A9DB0474145D69737D09CCDC63E1DEE7FE3744630440220143759437C04F7B61F012563AFE90D8DAFC46E86035E1D965A9CED282C97D4CE02204CFD241E86F17E011298FC1A39B63386C74306A5DE047E213B0F29EFA4571C2C8114DD76483FACDEE26E60D8A586BB58D09F27045C46 diff --git a/content/_code-samples/tx-serialization/test-cases/tx1-full.json b/content/_code-samples/tx-serialization/test-cases/tx1-full.json new file mode 100644 index 0000000000..fc02fcfae3 --- /dev/null +++ b/content/_code-samples/tx-serialization/test-cases/tx1-full.json @@ -0,0 +1,125 @@ +{ + "Account": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "Expiration": 595640108, + "Fee": "10", + "Flags": 524288, + "OfferSequence": 1752791, + "Sequence": 1752792, + "SigningPubKey": "03EE83BB432547885C219634A1BC407A9DB0474145D69737D09CCDC63E1DEE7FE3", + "TakerGets": "15000000000", + "TakerPays": { + "currency": "USD", + "issuer": "rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B", + "value": "7072.8" + }, + "TransactionType": "OfferCreate", + "TxnSignature": "30440220143759437C04F7B61F012563AFE90D8DAFC46E86035E1D965A9CED282C97D4CE02204CFD241E86F17E011298FC1A39B63386C74306A5DE047E213B0F29EFA4571C2C", + "hash": "73734B611DDA23D3F5F62E20A173B78AB8406AC5015094DA53F53D39B9EDB06C", + "metaData": { + "AffectedNodes": [ + { + "ModifiedNode": { + "FinalFields": { + "Flags": 0, + "IndexNext": "0000000000000000", + "IndexPrevious": "0000000000000000", + "Owner": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "RootIndex": "10AF5737F535F47CA9E8B6F82C4B7F4D998B1B7C44185C6078A22C751FD9FB7D" + }, + "LedgerEntryType": "DirectoryNode", + "LedgerIndex": "10AF5737F535F47CA9E8B6F82C4B7F4D998B1B7C44185C6078A22C751FD9FB7D" + } + }, + { + "DeletedNode": { + "FinalFields": { + "Account": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "BookDirectory": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10BFD011CB2800", + "BookNode": "0000000000000000", + "Expiration": 595640096, + "Flags": 131072, + "OwnerNode": "0000000000000000", + "PreviousTxnID": "50C2CBD3BEF831D80C2950D3001E67F1D257665569A9D77B1F0E0B8B4D178CEB", + "PreviousTxnLgrSeq": 43010795, + "Sequence": 1752791, + "TakerGets": "15000000000", + "TakerPays": { + "currency": "USD", + "issuer": "rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B", + "value": "7071.75" + } + }, + "LedgerEntryType": "Offer", + "LedgerIndex": "233E9A034C083E895EF7B4F6A643291FEF1608D55C8C5783F71E9C5F82D3E7FB" + } + }, + { + "ModifiedNode": { + "FinalFields": { + "Account": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "Balance": "37180946255", + "Flags": 0, + "OwnerCount": 6, + "Sequence": 1752793 + }, + "LedgerEntryType": "AccountRoot", + "LedgerIndex": "9EB65374048F2AED1995A6725D4234545432083B0C5728627E06443A8E1F4C98", + "PreviousFields": { + "Balance": "37180946265", + "Sequence": 1752792 + }, + "PreviousTxnID": "50C2CBD3BEF831D80C2950D3001E67F1D257665569A9D77B1F0E0B8B4D178CEB", + "PreviousTxnLgrSeq": 43010795 + } + }, + { + "DeletedNode": { + "FinalFields": { + "ExchangeRate": "4E10BFD011CB2800", + "Flags": 0, + "RootIndex": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10BFD011CB2800", + "TakerGetsCurrency": "0000000000000000000000000000000000000000", + "TakerGetsIssuer": "0000000000000000000000000000000000000000", + "TakerPaysCurrency": "0000000000000000000000005553440000000000", + "TakerPaysIssuer": "0A20B3C85F482532A9578DBB3950B85CA06594D1" + }, + "LedgerEntryType": "DirectoryNode", + "LedgerIndex": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10BFD011CB2800" + } + }, + { + "CreatedNode": { + "LedgerEntryType": "DirectoryNode", + "LedgerIndex": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10C0730D0B8000", + "NewFields": { + "ExchangeRate": "4E10C0730D0B8000", + "RootIndex": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10C0730D0B8000", + "TakerPaysCurrency": "0000000000000000000000005553440000000000", + "TakerPaysIssuer": "0A20B3C85F482532A9578DBB3950B85CA06594D1" + } + } + }, + { + "CreatedNode": { + "LedgerEntryType": "Offer", + "LedgerIndex": "FD6C2E2D72319FB0C22FC50B0AF993B2AF26717927EAEB1E8857971EE2C3CADD", + "NewFields": { + "Account": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "BookDirectory": "DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4E10C0730D0B8000", + "Expiration": 595640108, + "Flags": 131072, + "Sequence": 1752792, + "TakerGets": "15000000000", + "TakerPays": { + "currency": "USD", + "issuer": "rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B", + "value": "7072.8" + } + } + } + } + ], + "TransactionIndex": 0, + "TransactionResult": "tesSUCCESS" + } +} diff --git a/content/_code-samples/tx-serialization/test-cases/tx1-nometa.json b/content/_code-samples/tx-serialization/test-cases/tx1-nometa.json new file mode 100644 index 0000000000..987dd58aa7 --- /dev/null +++ b/content/_code-samples/tx-serialization/test-cases/tx1-nometa.json @@ -0,0 +1,18 @@ +{ + "Account": "rMBzp8CgpE441cp5PVyA9rpVV7oT8hP3ys", + "Expiration": 595640108, + "Fee": "10", + "Flags": 524288, + "OfferSequence": 1752791, + "Sequence": 1752792, + "SigningPubKey": "03EE83BB432547885C219634A1BC407A9DB0474145D69737D09CCDC63E1DEE7FE3", + "TakerGets": "15000000000", + "TakerPays": { + "currency": "USD", + "issuer": "rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B", + "value": "7072.8" + }, + "TransactionType": "OfferCreate", + "TxnSignature": "30440220143759437C04F7B61F012563AFE90D8DAFC46E86035E1D965A9CED282C97D4CE02204CFD241E86F17E011298FC1A39B63386C74306A5DE047E213B0F29EFA4571C2C", + "hash": "73734B611DDA23D3F5F62E20A173B78AB8406AC5015094DA53F53D39B9EDB06C" +} diff --git a/content/references/rippled-api/transaction-formats/transaction-common-fields.md b/content/references/rippled-api/transaction-formats/transaction-common-fields.md index 5e12440224..91f08f34e4 100644 --- a/content/references/rippled-api/transaction-formats/transaction-common-fields.md +++ b/content/references/rippled-api/transaction-formats/transaction-common-fields.md @@ -14,7 +14,7 @@ Every transaction has the same set of common fields, plus additional fields base | [Memos][] | Array of Objects | Array | _(Optional)_ Additional arbitrary information used to identify this transaction. | | [Signers][] | Array | Array | _(Optional)_ Array of objects that represent a [multi-signature](multi-signing.html) which authorizes this transaction. | | SourceTag | Unsigned Integer | UInt32 | _(Optional)_ Arbitrary integer used to identify the reason for this payment, or a sender on whose behalf this transaction is made. Conventionally, a refund should specify the initial payment's `SourceTag` as the refund payment's `DestinationTag`. | -| SigningPubKey | String | PubKey | _(Automatically added when signing)_ Hex representation of the public key that corresponds to the private key used to sign this transaction. If an empty string, indicates a multi-signature is present in the `Signers` field instead. | +| SigningPubKey | String | VariableLength | _(Automatically added when signing)_ Hex representation of the public key that corresponds to the private key used to sign this transaction. If an empty string, indicates a multi-signature is present in the `Signers` field instead. | | TxnSignature | String | VariableLength | _(Automatically added when signing)_ The signature that verifies this transaction as originating from the account it says it is from. | [auto-fillable]: #auto-fillable-fields diff --git a/content/references/rippled-api/transaction-formats/txserialization.md b/content/references/rippled-api/transaction-formats/txserialization.md index 761537df44..b63c2b0a8b 100644 --- a/content/references/rippled-api/transaction-formats/txserialization.md +++ b/content/references/rippled-api/transaction-formats/txserialization.md @@ -31,7 +31,25 @@ For example, the `Flags` [common transaction field](transaction-common-fields.ht All fields in a transaction are sorted in a specific order based on the field's type first, then the field itself second. (Think of it as sorting by family name, then given name, where the family name is the field's type and the given name is the field itself.) -When you combine the type code and sort code, you get the field's identifier, which is prefixed before the field in the final serialized blob. If both the type code and the field code +### Field IDs + +[[Source - Encoding]](https://github.com/seelabs/rippled/blob/cecc0ad75849a1d50cc573188ad301ca65519a5b/src/ripple/protocol/impl/Serializer.cpp#L117-L148 "Source") +[[Source - Decoding]](https://github.com/seelabs/rippled/blob/cecc0ad75849a1d50cc573188ad301ca65519a5b/src/ripple/protocol/impl/Serializer.cpp#L484-L509 "Source") + + +When you combine the type code and sort code, you get the field's identifier, which is prefixed before the field in the final serialized blob. The size of the Field ID is one to three bytes depending on the type code and field codes it combines. See the following table: + +| | Type Code < 16 | Type Code >= 16 | +|:-----------------|:------------------------------------------------------------------------------|:--| +| **Field Code < 16** | 1 byte: high 4 bits define type; low 4 bits define field. | 2 bytes: low 4 bits of the first byte define field; next byte defines type | +| **Field Code >= 16** | 2 bytes: high 4 bits of the first byte define type; low 4 bits of first byte are 0; next byte defines field | 3 bytes: first byte is 0x00, second byte defines type; third byte defines field | + +When decoding, you can tell how many bytes the field ID is by which bits **of the first byte** are zeroes. This corresponds to the cases in the above table: + +| | High 4 bits are nonzero | High 4 bits are zero | +|:-----------------|:------------------------------------------------------------------------------|:--| +| **Low 4 bits are nonzero** | 1 byte: high 4 bits define type; low 4 bits define field. | 2 bytes: low 4 bits of the first byte define field; next byte defines type | +| **Low 4 bits are zero** | 2 bytes: high 4 bits of the first byte define type; low 4 bits of first byte are 0; next byte defines field | 3 bytes: first byte is 0x00, second byte defines type; third byte defines field | ### Type Codes @@ -67,3 +85,22 @@ The "Amount" type is a special field type that represents an amount of currency, 3. 160 bits indicating the issuer's Account ID. (See also: [Account Address Encoding](accounts.html#address-encoding)) You can tell which of the two sub-types it is based on the first bit: `0` for XRP; `1` for issued currency. + +### VariableLength Fields + +The "VariableLength" type represents binary data of arbitrary length. The most important such fields are `SigningPubKey`, which every signed transaction includes to indicate the key pair used to sign the transaction, and `TxnSignature`, which contains the actual signature for any signed transaction. (Other uses of variable length fields include account `Domain` settings and the `Condition` and `Fulfillment` of conditional escrows.) + +VariableLength fields are encoded with one to three bytes indicating the length of the field immediately after the type prefix and before the contents. + +- If the VariableLength field contains 0 to 192 bytes of data, the first byte defines the length of the VariableLength data, then that many bytes of data follow immediately after the length byte. +- If the VariableLength field contains 193 to 12480 bytes of data, the first two bytes indicate the length of the field with the following formula: + 193 + ((byte1 - 193) * 256) + byte2 +- If the VariableLength field contains 12481 to 918744 bytes of data, the first three bytes indicate the length of the field with the following formula: + 12481 + ((byte1 - 241) * 65536) + (byte2 * 256) + byte3; +- A VariableLength field cannot contain more than 918744 bytes of data. + +When decoding, you can tell from the value of the first length byte whether there are 0, 1, or 2 more length bytes: + +- If the first length byte has a value of 192 or less, then that's the only length byte and it contains the exact length of the field contents in bytes. +- If the first length byte has a value of 193 to 240, then there are two length bytes. +- If the first length byte has a value of 241 to 254, then there are three length bytes.