Serialization: objects, arrays, hashes

content/_code-samples/tx-serialization/serialize.py

@@ -39,6 +39,11 @@ def field_sort_key(field_name):
     return (DEFINITIONS["TYPES"][field_type_name], DEFINITIONS["FIELDS"][field_name]["nth"])
 
 def field_id(field_name):
+    """
+    Returns the unique field ID for a given field name.
+    This field ID consists of the type code and field code, in 1 to 3 bytes
+    depending on whether those values are "common" (<16) or uncommon (>=16)
+    """
     field_type_name = DEFINITIONS["FIELDS"][field_name]["type"]
     type_code = DEFINITIONS["TYPES"][field_type_name]
     field_code = DEFINITIONS["FIELDS"][field_name]["nth"]
@@ -145,10 +150,59 @@ def vl_to_bytes(field_val):
     vl_contents = bytes.fromhex(field_val)
     return vl_encode(vl_contents)
 
+def hash_to_bytes(contents):
+    return bytes.fromhex(field_val)
+
 def accountid_to_bytes(address):
     return vl_encode(decode_address(address))
 
+def array_to_bytes(array):
+    """
+    Serialize an array of objects.
+    Each member object must have a type wrapper and an inner object.
+    """
+    members_as_bytes = []
+    for el in array:
+        wrapper_key = list(el.keys())[0]
+        inner_obj = el[wrapper_key]
+        members_as_bytes.append(field_to_bytes(field_name=wrapper_key, field_val=el))
+    members_as_bytes.append(field_id("ArrayEndMarker"))
+    return b''.join(members_as_bytes)
+
+def object_to_bytes(obj):
+    """
+    Serialize an object, assuming a type wrapper, for example:
+
+    {
+        "SignerEntry": {
+            "Account": "rUpy3eEg8rqjqfUoLeBnZkscbKbFsKXC3v",
+            "SignerWeight": 1
+        }
+    }
+
+    Puts the child fields (e.g. Account, SignerWeight) in canonical order
+    and appends an object end marker.
+    """
+    wrapper_key = list(obj.keys())[0]
+    inner_obj = obj[wrapper_key]
+    child_order = sorted(inner_obj.keys(), key=field_sort_key)
+    fields_as_bytes = []
+    for field_name in child_order:
+        if (DEFINITIONS["FIELDS"][field_name]["isSerialized"]):
+            field_val = inner_obj[field_name]
+            field_bytes = field_to_bytes(field_name, field_val)
+            logger.debug("{n}: {h}".format(n=field_name, h=field_bytes.hex()))
+            fields_as_bytes.append(field_bytes)
+
+    fields_as_bytes.append(field_id("ObjectEndMarker"))
+    return b''.join(fields_as_bytes)
+
+
 def field_to_bytes(field_name, field_val):
+    """
+    Returns a bytes object containing the serialized version of a field
+    including its field ID prefix.
+    """
     field_type = DEFINITIONS["FIELDS"][field_name]["type"]
     logger.debug("Serializing field {f} of type {t}".format(f=field_name, t=field_type))
 
@@ -161,13 +215,19 @@ def field_to_bytes(field_name, field_val):
 
     dispatch = {
         # TypeName: function(field): bytes object
-        "UInt64": lambda x:bytes_from_uint(x, 64),
-        "UInt32": lambda x:bytes_from_uint(x, 32),
-        "UInt16": lambda x:bytes_from_uint(x, 16),
-        "UInt8" : lambda x:bytes_from_uint(x, 8),
         "AccountID": accountid_to_bytes,
         "Amount": amount_to_bytes,
-        "Blob": vl_to_bytes
+        "Blob": vl_to_bytes,
+        "Hash128": hash_to_bytes,
+        "Hash160": hash_to_bytes,
+        "Hash256": hash_to_bytes,
+        # TODO: PathSet
+        "STArray": array_to_bytes,
+        "STObject": object_to_bytes,
+        "UInt8" : lambda x:bytes_from_uint(x, 8),
+        "UInt16": lambda x:bytes_from_uint(x, 16),
+        "UInt32": lambda x:bytes_from_uint(x, 32),
+        "UInt64": lambda x:bytes_from_uint(x, 64),
     }
     field_binary = dispatch[field_type](field_val)
     return b''.join( (id_prefix, field_binary) )
@@ -210,7 +270,7 @@ if __name__ == "__main__":
     #   "Sequence": 2
     # }
 
-    with open("test-cases/tx1-nometa.json") as f:
+    with open("test-cases/tx2-nometa.json") as f:
         example_tx = json.load(f)
 
     serialize_tx(example_tx)

content/_code-samples/tx-serialization/test-cases/tx2-binary.txt

@@ -0,0 +1 @@
+1200022280000000240000000120190000000B68400000000000277573210268D79CD579D077750740FA18A2370B7C2018B2714ECE70BA65C38D223E79BC9C74473045022100F06FB54049D6D50142E5CF2E2AC21946AF305A13E2A2D4BA881B36484DD01A540220311557EC8BEF536D729605A4CB4D4DC51B1E37C06C93434DD5B7651E1E2E28BF811452C7F01AD13B3CA9C1D133FA8F3482D2EF08FA7D82145A380FBD236B6A1CD14B939AD21101E5B6B6FFA2F9EA7D0F04C4D46544659A2D58525043686174E1F1

content/_code-samples/tx-serialization/test-cases/tx2-full.json

@@ -0,0 +1,94 @@
+{
+    "hash": "C0B450C8601E19CB0BDED71C4B523B2A4AAB77415B08E7923D8DA3F831631702",
+    "ledger_index": 36259236,
+    "date": "2018-02-01T09:45:32+00:00",
+    "tx": {
+        "TransactionType": "EscrowFinish",
+        "Flags": 2147483648,
+        "Sequence": 1,
+        "OfferSequence": 11,
+        "Fee": "10101",
+        "SigningPubKey": "0268D79CD579D077750740FA18A2370B7C2018B2714ECE70BA65C38D223E79BC9C",
+        "TxnSignature": "3045022100F06FB54049D6D50142E5CF2E2AC21946AF305A13E2A2D4BA881B36484DD01A540220311557EC8BEF536D729605A4CB4D4DC51B1E37C06C93434DD5B7651E1E2E28BF",
+        "Account": "r3Y6vCE8XqfZmYBRngy22uFYkmz3y9eCRA",
+        "Owner": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3",
+        "Memos": [
+            {
+                "Memo": {
+                    "MemoData": "04C4D46544659A2D58525043686174"
+                }
+            }
+        ]
+    },
+    "meta": {
+        "TransactionIndex": 35,
+        "AffectedNodes": [
+            {
+                "DeletedNode": {
+                    "LedgerEntryType": "Escrow",
+                    "LedgerIndex": "983EBDF89C1C30CECDB105B593E7DEBE602AF89012EDB4DD76D24ACEF92C89EF",
+                    "FinalFields": {
+                        "Flags": 0,
+                        "PreviousTxnLgrSeq": 35059511,
+                        "FinishAfter": 570758400,
+                        "OwnerNode": "0000000000000000",
+                        "PreviousTxnID": "6F54E04D7B205CBE2FCAEF1C249E62A9759721C7FE1F6992FD800266C8E4814C",
+                        "Amount": "1000000000000000",
+                        "Account": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3",
+                        "Destination": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3"
+                    }
+                }
+            },
+            {
+                "ModifiedNode": {
+                    "LedgerEntryType": "DirectoryNode",
+                    "LedgerIndex": "C05DC35EBDCA5D8697190FF41950EC5AFCBF7F61C1177DC73000CC17C2149886",
+                    "FinalFields": {
+                        "Flags": 0,
+                        "RootIndex": "C05DC35EBDCA5D8697190FF41950EC5AFCBF7F61C1177DC73000CC17C2149886",
+                        "Owner": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3"
+                    }
+                }
+            },
+            {
+                "ModifiedNode": {
+                    "LedgerEntryType": "AccountRoot",
+                    "PreviousTxnLgrSeq": 36230933,
+                    "PreviousTxnID": "9366F75CD350ACE0EEFC0A392ECBD5AC2B84C06E5DEC2DE895B76FFC7BD55553",
+                    "LedgerIndex": "C180CA555CE8820D8F1086CDCA756FF8D5813FEB2AE3FF002B9F48F870CA08A0",
+                    "PreviousFields": {
+                        "Sequence": 1,
+                        "Balance": "26000000"
+                    },
+                    "FinalFields": {
+                        "Flags": 0,
+                        "Sequence": 2,
+                        "OwnerCount": 0,
+                        "Balance": "25989899",
+                        "Account": "r3Y6vCE8XqfZmYBRngy22uFYkmz3y9eCRA"
+                    }
+                }
+            },
+            {
+                "ModifiedNode": {
+                    "LedgerEntryType": "AccountRoot",
+                    "PreviousTxnLgrSeq": 36253527,
+                    "PreviousTxnID": "F2AA5584005847C19F59A9C87E7BF3108F97F2567C5083159EEC40B08ED90F46",
+                    "LedgerIndex": "DCED5AAFE87AA8D00E651DBBFBA2F992927BC3DC5FFD905EF49019ED02824B3A",
+                    "PreviousFields": {
+                        "OwnerCount": 14,
+                        "Balance": "200923000"
+                    },
+                    "FinalFields": {
+                        "Flags": 1048576,
+                        "Sequence": 18,
+                        "OwnerCount": 13,
+                        "Balance": "1000000200923000",
+                        "Account": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3"
+                    }
+                }
+            }
+        ],
+        "TransactionResult": "tesSUCCESS"
+    }
+}

content/_code-samples/tx-serialization/test-cases/tx2-nometa.json

@@ -0,0 +1,18 @@
+{
+    "TransactionType": "EscrowFinish",
+    "Flags": 2147483648,
+    "Sequence": 1,
+    "OfferSequence": 11,
+    "Fee": "10101",
+    "SigningPubKey": "0268D79CD579D077750740FA18A2370B7C2018B2714ECE70BA65C38D223E79BC9C",
+    "TxnSignature": "3045022100F06FB54049D6D50142E5CF2E2AC21946AF305A13E2A2D4BA881B36484DD01A540220311557EC8BEF536D729605A4CB4D4DC51B1E37C06C93434DD5B7651E1E2E28BF",
+    "Account": "r3Y6vCE8XqfZmYBRngy22uFYkmz3y9eCRA",
+    "Owner": "r9NpyVfLfUG8hatuCCHKzosyDtKnBdsEN3",
+    "Memos": [
+        {
+            "Memo": {
+                "MemoData": "04C4D46544659A2D58525043686174"
+            }
+        }
+    ]
+}

content/references/rippled-api/api-conventions/serialization.md

@@ -18,12 +18,6 @@ The result is a single binary blob that can be signed using well-known signature
 
 **Note:** The XRP Ledger uses the same serialization format to represent other types of data, such as [ledger objects](ledger-object-types.html) and processed transactions. However, only certain fields are appropriate for including in a transaction that gets signed. (For example, the `TxnSignature` field, containing the signature itself, should not be present in the binary blob that you sign.) Thus, some fields are designated as "Signing" fields, which are included in objects when those objects are signed, and "non-signing" fields, which are not.
 
-The hard work is the details of each of those steps.
-
-***Notes: some useful links for signing:***
-
-- Actual core of the signing code in rippled: https://github.com/ripple/rippled/blob/develop/src/ripple/protocol/impl/STObject.cpp#L697-L718
-- Serialization code in ripple-lib depends on `ripple-binary-codec`. These definitions have the canonical types and sort codes for all fields: https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json
 
 ## Internal Format
 
@@ -31,6 +25,36 @@ Each field has an "internal" binary format used in the `rippled` source code to
 
 For example, the `Flags` [common transaction field](transaction-common-fields.html) becomes a UInt32 (32-bit unsigned integer).
 
+### Definitions File
+
+The following JSON file defines the important constants you need for serializing XRP Ledger data to its binary format and deserializing it from binary:
+
+**<https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json>**
+
+The following table defines the top-level fields from the definitions file:
+
+| Field                 | Contents                                             |
+|:----------------------|:-----------------------------------------------------|
+| `TYPES`               | Map of data types to their "type code" for constructing field IDs and sorting fields in canonical order. Codes below 1 should not appear in actual data; codes above 10000 represent special "high-level" object types such as "Transaction" that cannot be serialized inside other objects. |
+| `LEDGER_ENTRY_TYPES`  | Map of [ledger objects](ledger-object-types.html) to their data type. These appear in ledger state data, and in the "affected nodes" section of processed transactions' [metadata](transaction-metadata.html). |
+| `FIELDS`              | A sorted array of tuples representing all fields that may appear in transactions, ledger objects, or other data. The first member of each tuple is the string name of the field and the second member is an object with that field's properties. (See the "Field properties" table below for definitions of those fields.) |
+| `TRANSACTION_RESULTS` | Map of [transaction result codes](transaction-results.html) to their numeric values. Result types not included in ledgers have negative values;`tesSUCCESS` has numeric value 0; [`tec`-class codes](tec-codes.html) represent failures that are included in ledgers. |
+| `TRANSACTION_TYPES`   | Map of all [transaction types](transaction-types.html) to their numeric values. |
+
+For purposes of serializing transactions for signing and submitting, only `FIELDS` and `TYPES` are necessary.
+
+The field definition objects in the `FIELDS` array have the following fields:
+
+| Field            | Type    | Contents                                        |
+|:-----------------|:--------|:------------------------------------------------|
+| `nth`            | Number  | The [field code](#field-codes) of this field, for use in constructing its [Field ID](#field-ids) and ordering it with other fields of the same data type. |
+| `isVLEncoded`    | Boolean | If `true`, this field is [variable-length encoded](#variable-length-encoding). |
+| `isSerialized`   | Boolean | If `true`, this field should be encoded into serialized binary data. When this field is `false`, the field is typically reconstructed on demand rather than stored. |
+| `isSigningField` | Boolean | If `true` this field should be serialized when preparing a transaction for signing. If `false`, this field should be omitted from the data to be signed. (It may not be part of transactions at all.) |
+| `type`           | String  | The internal data type of this field. This maps to a key in the `TYPES` map, which gives the [type code](#type-codes) for this field. |
+
+
+
 ## Canonical Field Order
 
 All fields in a transaction are sorted in a specific order based on the field's type first, then the field itself second. (Think of it as sorting by family name, then given name, where the family name is the field's type and the given name is the field itself.)
@@ -149,9 +173,11 @@ You can tell which of the two sub-types it is based on the first bit: `0` for XR
 ### Array Fields
 [STArray]: #array-fields
 
-Some transaction fields, such as `SignerEntries` (in [SignerListSet transactions][]) and [`Memos`](transaction-common-fields.html#memos-field), are arrays.
+Some transaction fields, such as `SignerEntries` (in [SignerListSet transactions][]) and [`Memos`](transaction-common-fields.html#memos-field), are arrays of objects (called the "STArray" type).
 
-Arrays contain several other fields in their native binary format in a specific order. Each of these fields has its normal Field ID prefix and contents. To mark the end of an array, append an item with a "Field ID" of `0xf1` (the type code for array with field code of 1) and no contents.
+Arrays contain several [object fields](#object-fields) in their native binary format in a specific order. In JSON, each array member is a JSON "wrapper" object with a single field, which is the name of the member object field. The value of that field is the ("inner") object itself.
+
+In the binary format, each member of the array has a Field ID prefix (based on the single key of the wrapper object) and contents (comprising the inner object, [serialized as an object](#object-fields)). To mark the end of an array, append an item with a "Field ID" of `0xf1` (the type code for array with field code of 1) and no contents.
 
 The following example shows the serialization format for an array (the `SignerEntries` field):
 
@@ -166,10 +192,20 @@ The Blob type is a [variable-length encoded](#variable-length-encoding) field wi
 Blob fields have no further structure to their contents, so they consist of exactly the amount of bytes indicated in the variable-length encoding, after the Field ID and length prefixes.
 
 
+### Hash Fields
+[Hash128]: #hash-fields
+[Hash160]: #hash-fields
+[Hash256]: #hash-fields
+
+The XRP Ledger has several "hash" types: Hash128, Hash160, and Hash256. These fields contain arbitrary binary data of the given number of bits, which may or may not represent the result of a hash operation.
+
+All such fields are serialized as the specific number of bits, with no length indicator, in big-endian byte order.
+
+
 ### Object Fields
 [STObject]: #object-fields
 
-Some fields, such as `SignerEntry` (in [SignerListSet transactions][]), and `Memo` (in `Memos` arrays) are objects. The serialization of objects is very similar to that of arrays, with one difference: **object members must be placed in canonical order** within the object field, where array fields have an explicit order already.
+Some fields, such as `SignerEntry` (in [SignerListSet transactions][]), and `Memo` (in `Memos` arrays) are objects (called the "STObject" type). The serialization of objects is very similar to that of arrays, with one difference: **object members must be placed in canonical order** within the object field, where array fields have an explicit order already.
 
 The [canonical field order](#canonical-field-order) of object fields is the same as the canonical field order for all top-level fields, but the members of the object must be sorted within the object. After the last member, there is an "Object end" Field ID of `0xe1` with no contents.
 
@@ -177,6 +213,7 @@ The following example shows the serialization format for an object (a single `Me
 
 ![Object field ID, followed by the Object ID and contents of each object member in canonical order, followed by the "Object end" field ID](img/serialization-object.png)
 
+
 ### UInt Fields
 [UInt8]: #uint-fields
 [UInt16]: #uint-fields