Serialization - sample code in progress, clarifications

content/_code-samples/tx-serialization/address.py

@@ -0,0 +1,8 @@
+import base58.base58 as base58
+
+def decode_address(address):
+    decoded = base58.b58decode_check(address)
+    if decoded[0] == 0 and len(decoded) == 21: # is an address
+        return decoded[1:]
+    else:
+        raise ValueError("Not an AccountID!")

content/_code-samples/tx-serialization/base58/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2015 David Keijser
+
+	Permission is hereby granted, free of charge, to any person obtaining a copy
+	of this software and associated documentation files (the "Software"), to deal
+	in the Software without restriction, including without limitation the rights
+	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+	copies of the Software, and to permit persons to whom the Software is
+	furnished to do so, subject to the following conditions:
+
+	The above copyright notice and this permission notice shall be included in
+	all copies or substantial portions of the Software.
+
+	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+	THE SOFTWARE.

content/_code-samples/tx-serialization/base58/base58.py

@@ -0,0 +1,177 @@
+'''Base58 encoding
+
+Implementations of Base58 and Base58Check endcodings that are compatible
+with the XRP Ledger.
+'''
+
+# This This code is adapted from the module by David Keijser at
+# <https://github.com/keis/base58>. - rome@ripple.com
+# His notes are preserved below:
+
+# This module is based upon base58 snippets found scattered over many bitcoin
+# tools written in python. From what I gather the original source is from a
+# forum post by Gavin Andresen, so direct your praise to him.
+# This module adds shiny packaging and support for python3.
+
+from hashlib import sha256
+
+__version__ = '1.0.3-xrp'
+
+# 58 character alphabet used
+# alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' # Bitcoin
+alphabet = b'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz' # XRP Ledger
+
+
+if bytes == str:  # python2
+    iseq, bseq, buffer = (
+        lambda s: map(ord, s),
+        lambda s: ''.join(map(chr, s)),
+        lambda s: s,
+    )
+else:  # python3
+    iseq, bseq, buffer = (
+        lambda s: s,
+        bytes,
+        lambda s: s.buffer,
+    )
+
+
+def scrub_input(v):
+    if isinstance(v, str) and not isinstance(v, bytes):
+        v = v.encode('ascii')
+
+    if not isinstance(v, bytes):
+        raise TypeError(
+            "a bytes-like object is required (also str), not '%s'" %
+            type(v).__name__)
+
+    return v
+
+
+def b58encode_int(i, default_one=True):
+    '''Encode an integer using Base58'''
+    if not i and default_one:
+        return alphabet[0:1]
+    string = b""
+    while i:
+        i, idx = divmod(i, 58)
+        string = alphabet[idx:idx+1] + string
+    return string
+
+
+def b58encode(v):
+    '''Encode a string using Base58'''
+
+    v = scrub_input(v)
+
+    nPad = len(v)
+    v = v.lstrip(b'\0')
+    nPad -= len(v)
+
+    p, acc = 1, 0
+    for c in iseq(reversed(v)):
+        acc += p * c
+        p = p << 8
+
+    result = b58encode_int(acc, default_one=False)
+
+    return (alphabet[0:1] * nPad + result)
+
+
+def b58decode_int(v):
+    '''Decode a Base58 encoded string as an integer'''
+
+    v = scrub_input(v)
+
+    decimal = 0
+    for char in v:
+        decimal = decimal * 58 + alphabet.index(char)
+    return decimal
+
+
+def b58decode(v):
+    '''Decode a Base58 encoded string'''
+
+    v = scrub_input(v)
+
+    origlen = len(v)
+    v = v.lstrip(alphabet[0:1])
+    newlen = len(v)
+
+    acc = b58decode_int(v)
+
+    result = []
+    while acc > 0:
+        acc, mod = divmod(acc, 256)
+        result.append(mod)
+
+    return (b'\0' * (origlen - newlen) + bseq(reversed(result)))
+
+
+def b58encode_check(v):
+    '''Encode a string using Base58 with a 4 character checksum'''
+
+    digest = sha256(sha256(v).digest()).digest()
+    return b58encode(v + digest[:4])
+
+
+def b58decode_check(v):
+    '''Decode and verify the checksum of a Base58 encoded string'''
+
+    result = b58decode(v)
+    result, check = result[:-4], result[-4:]
+    digest = sha256(sha256(result).digest()).digest()
+
+    if check != digest[:4]:
+        raise ValueError("Invalid checksum")
+
+    return result
+
+
+def main():
+    '''Base58 encode or decode FILE, or standard input, to standard output.'''
+
+    import sys
+    import argparse
+
+    stdout = buffer(sys.stdout)
+
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument(
+        'file',
+        metavar='FILE',
+        nargs='?',
+        type=argparse.FileType('r'),
+        default='-')
+    parser.add_argument(
+        '-d', '--decode',
+        action='store_true',
+        help='decode data')
+    parser.add_argument(
+        '-c', '--check',
+        action='store_true',
+        help='append a checksum before encoding')
+
+    args = parser.parse_args()
+    fun = {
+        (False, False): b58encode,
+        (False, True): b58encode_check,
+        (True, False): b58decode,
+        (True, True): b58decode_check
+    }[(args.decode, args.check)]
+
+    data = buffer(args.file).read()
+
+    try:
+        result = fun(data)
+    except Exception as e:
+        sys.exit(e)
+
+    if not isinstance(result, bytes):
+        result = result.encode('ascii')
+
+    stdout.write(result)
+
+
+if __name__ == '__main__':
+    main()

content/_code-samples/tx-serialization/field_ordering.py

@@ -0,0 +1,180 @@
+# Canonical ordering (of transaction signing fields only)
+# for serializing and signing transactions
+
+TYPES = {
+    # Type name: canonical order for this type
+    "UInt16": 1,
+    "UInt32": 2,
+    "UInt64": 3,
+    "Hash128": 4,
+    "Hash256": 5,
+    "Amount": 6,
+    "VL": 7, # VariableLength
+    "Account": 8,
+    # 9-13 are reserved
+    "Object": 14,
+    "Array": 15,
+    "UInt8": 16,
+    "Hash160": 17,
+    "PathSet": 18, # "Paths" field of cross-currency payments only
+    "Vector256": 19,
+}
+
+FIELDS = {
+    # Type name: Canonical order for this field (among fields of its type)
+
+    # UInt16 types
+    "LedgerEntryType": 1,
+    "TransactionType": 2,
+    "SignerWeight": 3,
+    # UInt32 types
+    "Flags": 2,
+    "SourceTag": 3,
+    "Sequence": 4,
+    "PreviousTxnLgrSeq": 5,
+    "LedgerSequence": 6,
+    "CloseTime": 7,
+    "ParentCloseTime": 8,
+    "SigningTime": 9,
+    "Expiration": 10,
+    "TransferRate": 11,
+    "WalletSize": 12,
+    "OwnerCount": 13,
+    "DestinationTag": 14,
+    "HighQualityIn": 16,
+    "HighQualityOut": 17,
+    "LowQualityIn": 18,
+    "LowQualityOut": 19,
+    "QualityIn": 20,
+    "QualityOut": 21,
+    "StampEscrow": 22,
+    "BondAmount": 23,
+    "LoadFee": 24,
+    "OfferSequence": 25,
+    "FirstLedgerSequence": 26,
+    "LastLedgerSequence": 27,
+    "TransactionIndex": 28,
+    "OperationLimit": 29,
+    "ReferenceFeeUnits": 30,
+    "ReserveBase": 31,
+    "ReserveIncrement": 32,
+    "SetFlag": 33,
+    "ClearFlag": 34,
+    "SignerQuorum": 35,
+    "CancelAfter": 36,
+    "FinishAfter": 37,
+    "SignerListID": 38,
+    "SettleDelay": 39,
+    # UInt64 types
+    "IndexNext": 1,
+    "IndexPrevious": 2,
+    "BookNode": 3,
+    "OwnerNode": 4,
+    "BaseFee": 5,
+    "ExchangeRate": 6,
+    "LowNode": 7,
+    "HighNode": 8,
+    "DestinationNode": 9,
+    "Cookie": 10,
+    # Hash128 types
+    "EmailHash": 1,
+    # Hash256 types
+    "LedgerHash": 1,
+    "ParentHash": 2,
+    "TransactionHash": 3,
+    "AccountHash": 4,
+    "PreviousTxnID": 5,
+    "LedgerIndex": 6,
+    "WalletLocator": 7,
+    "RootIndex": 8,
+    "AccountTxnID": 9,
+    "BookDirectory": 16,
+    "InvoiceID": 17,
+    "Nickname": 18,
+    "Amendment": 19,
+    "TicketID": 20,
+    "Digest": 21,
+    "Channel": 22,
+    "ConsensusHash": 23,
+    "CheckID": 24,
+    "hash": 257,
+    "index": 258,
+    # Amount types
+    "Amount": 1,
+    "Balance": 2,
+    "LimitAmount": 3,
+    "TakerPays": 4,
+    "TakerGets": 5,
+    "LowLimit": 6,
+    "HighLimit": 7,
+    "Fee": 8,
+    "SendMax": 9,
+    "DeliverMin": 10,
+    "MinimumOffer": 16,
+    "RippleEscrow": 17,
+    "DeliveredAmount": 18,
+    # VL types
+    "PublicKey": 1,
+    "MessageKey": 2,
+    "SigningPubKey": 3,
+    "TxnSignature": 4,
+    "Signature": 6,
+    "Domain": 7,
+    "FundCode": 8,
+    "RemoveCode": 9,
+    "ExpireCode": 10,
+    "CreateCode": 11,
+    "MemoType": 12,
+    "MemoData": 13,
+    "MemoFormat": 14,
+    "Fulfillment": 16,
+    "Condition": 17,
+    "MasterSignature": 18,
+    # Account types
+    "Account": 1,
+    "Owner": 2,
+    "Destination": 3,
+    "Issuer": 4,
+    "Authorize": 5,
+    "Unauthorize": 6,
+    "Target": 7,
+    "RegularKey": 8,
+    # Object types
+    "TransactionMetaData": 2,
+    "CreatedNode": 3,
+    "DeletedNode": 4,
+    "ModifiedNode": 5,
+    "PreviousFields": 6,
+    "FinalFields": 7,
+    "NewFields": 8,
+    "TemplateEntry": 9,
+    "Memo": 10,
+    "SignerEntry": 11,
+    "Signer": 16,
+    "Majority": 18,
+    # Array types
+    "Signers": 3,
+    "SignerEntries": 4,
+    "Template": 5,
+    "Necessary": 6,
+    "Sufficient": 7,
+    "AffectedNodes": 8,
+    "Memos": 9,
+    "Majorities": 16,
+    # UInt8 types
+    "CloseResolution": 1,
+    "Method": 2,
+    "TransactionResult": 3,
+    "TickSize": 16,
+    # Hash160 types
+    "TakerPaysCurrency": 1,
+    "TakerPaysIssuer": 2,
+    "TakerGetsCurrency": 3,
+    "TakerGetsIssuer": 4,
+    # PathSet types
+    "Paths": 1,
+    # Vector256 types
+    "Indexes": 1,
+    "Hashes": 2,
+    "Amendments": 3,
+}

content/_code-samples/tx-serialization/serialize.py

@@ -0,0 +1,154 @@
+#!/bin/env python3
+
+# Transaction Serialization Sample Code (Python3 version)
+# Author: rome@ripple.com
+# Copyright Ripple 2018
+
+import json
+import logging
+
+from address import decode_address
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+
+def load_defs(fname="definitions.json"):
+    with open(fname) as definitions_file:
+        definitions = json.load(definitions_file)
+        return {
+            "TYPES": definitions["TYPES"],
+                # type_name str: type_sort_key int
+            "FIELDS": {k:v for (k,v) in definitions["FIELDS"]}, # convert list of tuples to dict
+                # field_name str: {
+                #   nth: field_sort_key int,
+                #   isVLEncoded: bool,
+                #   isSerialized: bool,
+                #   isSigningField: bool,
+                #   type: type_name str
+                # }
+            "LEDGER_ENTRY_TYPES": definitions["LEDGER_ENTRY_TYPES"],
+            "TRANSACTION_RESULTS": definitions["TRANSACTION_RESULTS"],
+            "TRANSACTION_TYPES": definitions["TRANSACTION_TYPES"],
+        }
+
+
+def field_sort_key(field_name):
+    """Return a tuple sort key for a given field name"""
+    field_type_name = DEFINITIONS["FIELDS"][field_name]["type"]
+    return (DEFINITIONS["TYPES"][field_type_name], DEFINITIONS["FIELDS"][field_name]["nth"])
+
+def field_id(field_name):
+    field_type_name = DEFINITIONS["FIELDS"][field_name]["type"]
+    type_code = DEFINITIONS["TYPES"][field_type_name]
+    field_code = DEFINITIONS["FIELDS"][field_name]["nth"]
+
+    if type_code < 16 and field_code < 16:
+        # first 4 bits is the type_code, next 4 bits is the field code
+        combined_code = (type_code << 4) | field_code
+        return combined_code.to_bytes(1, byteorder="big", signed=False)
+    else:
+        # TODO: need more than 1 byte to encode this field id
+        raise ValueError("field_id not yet implemented for types/fields > 16")
+
+def bytes_from_uint(i, bits):
+    if bits % 8:
+        raise ValueError("bytes_from_uint: bits must be divisible by 8")
+    return i.to_bytes(bits // 8, byteorder="big", signed=False)
+
+def amount_to_bytes(a):
+    if type(a) == str:
+        # is XRP
+        xrp_amt = int(a)
+        if (xrp_amt >= 0):
+            # set the "is positive" bit -- this is backwards from usual two's complement!
+            xrp_amt = xrp_amt | 0x4000000000000000
+        else:
+            # convert to absolute value, leaving the "is positive" bit unset
+            xrp_amt = -xrp_amt
+        return xrp_amt.to_bytes(8, byteorder="big", signed=False)
+    elif type(a) == dict:
+        if sorted(a.keys()) != ["currency", "issuer", "value"]:
+            raise ValueError("amount must have currency, value, issuer only (actually had: %s)" %
+                    sorted(a.keys()))
+        #TODO: canonicalize mantissa/exponent/etc. of issued currency amount
+        # https://developers.ripple.com/currency-formats.html#issued-currency-math
+        # temporarily returning all zeroes
+        issued_amt = bytes(8)
+        # TODO: calculate 160-bit currency ID
+        currency_code = bytes(20)
+        return issued_amt + currency_code + decode_address(a["issuer"])
+    else:
+        raise ValueError("amount must be XRP string or {currency, value, issuer}")
+
+def tx_type_to_bytes(txtype):
+    type_uint = DEFINITIONS["TRANSACTION_TYPES"][txtype]
+    return type_uint.to_bytes(2, byteorder="big", signed=False)
+
+def field_to_bytes(field_name, field_val):
+    field_type = DEFINITIONS["FIELDS"][field_name]["type"]
+    logger.debug("Serializing field {f} of type {t}".format(f=field_name, t=field_type))
+
+    id_prefix = field_id(field_name)
+    logger.debug("id_prefix is: %s" % id_prefix.hex())
+
+    if field_name == "TransactionType":
+        # Special case: convert from string to UInt16
+        return b''.join( (id_prefix, tx_type_to_bytes(field_val)) )
+
+    dispatch = {
+        # TypeName: function(field): bytes object
+        "UInt64": lambda x:bytes_from_uint(x, 64),
+        "UInt32": lambda x:bytes_from_uint(x, 32),
+        "UInt16": lambda x:bytes_from_uint(x, 16),
+        "UInt8" : lambda x:bytes_from_uint(x, 8),
+        "AccountID": decode_address,
+        "Amount": amount_to_bytes
+    }
+    field_binary = dispatch[field_type](field_val)
+    return b''.join( (id_prefix, field_binary) )
+
+def serialize_tx(tx):
+    field_order = sorted(tx.keys(), key=field_sort_key)
+    logger.debug("Canonical field order: %s" % field_order)
+
+    fields_as_bytes = []
+    for field_name in field_order:
+        field_val = tx[field_name]
+        fields_as_bytes.append(field_to_bytes(field_name, field_val))
+
+    all_serial = b''.join(fields_as_bytes)
+    logger.info(all_serial.hex().upper())
+    return all_serial
+
+
+
+################################################################################
+
+if __name__ == "__main__":
+    logger.setLevel(logging.DEBUG)
+    DEFINITIONS = load_defs()
+
+    example_tx = {
+      "TransactionType" : "Payment",
+      "Account" : "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination" : "ra5nK24KXen9AHvsdFTKHSANinZseWnPcX",
+      "Amount" : {
+         "currency" : "USD",
+         "value" : "1",
+         "issuer" : "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn"
+      },
+      "Fee": "12",
+      "Flags": 2147483648,
+      "Sequence": 2
+    }
+
+    serialize_tx(example_tx)
+
+    # example rippled signature:
+    # ./rippled sign masterpassphrase (the above JSON)
+    # where "masterpassphrase" is the key behind rHb9...
+    # snoPBrXtMeMyMHUVTgbuqAfg1SUTb in base58
+    # "tx_blob" : "1200002280000000240000000261D4838D7EA4C6800000000000000000000000000055534400000000004B4E9C06F24296074F7BC48F92A97916C6DC5EA968400000000000000C73210330E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD0207446304402201FE0A74FC1BDB509C8F42B861EF747C43B92917706BB623F0A0D621891933AF402205206FBA8B0BF6733DB5B03AD76B5A76A2D46DF9093916A3BEC78897E58A3DF148114B5F762798A53D543A014CAF8B297CFF8F2F937E883143E9D4A2B8AA0780F682D136F7A56D6724EF53754",
+    # "SigningPubKey" : "0330E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD020",
+    # "TxnSignature" : "304402201FE0A74FC1BDB509C8F42B861EF747C43B92917706BB623F0A0D621891933AF402205206FBA8B0BF6733DB5B03AD76B5A76A2D46DF9093916A3BEC78897E58A3DF14",
+    # "hash" : "8BA1509E4FB80CCF76CD9DE924B8B71597637C775BA2DC515F90C333DA534BF3"

content/references/rippled-api/transaction-formats/txserialization.md

@@ -5,17 +5,16 @@ XRP Ledger transactions have a canonical binary format, which is necessary to cr
 
 The process of serializing a transaction from JSON or any other representation into their canonical binary format can be summarized with these steps:
 
-1. Convert each field's data into its "internal" binary format.
-2. Sort the fields in canonical order.
-3. Concatenate the fields in their sorted order. ***TODO: seems there must be some sort of wrapping/control data to indicate which fields are present for cases with optional fields.***
+1. Make sure all required fields are provided, including any required but "auto-fillable" fields.
+    **Note:** The `SigningPubKey` must also be provided at this step. When signing, you can derive this key from the secret key that is provided for signing.
+2. Convert each field's data into its "internal" binary format.
+3. Sort the fields in canonical order.
+4. Prefix each field with an identifier.
+5. Concatenate the fields (including prefixes) in their sorted order.
 
-When serializing transaction instructions to be signed, you must also:
+The result is a single binary blob that can be signed using well-known signature algorithms such as ECDSA (with the secp256k1 elliptic curve) and Ed25519. After signing, you must attach the signature to the transaction, calculate the transaction's identifying hash, then re-serialize the transaction with the additional fields.
 
-- Make sure all required fields are provided, including any required but "auto-fillable" fields.
-- Add the `SigningPubKey` field ***TODO: at what point in the above process? Probably before sorting, if it's a signing field?***
-
-
-The result is a single binary blob that can be signed using well-known signature algorithms such as ECDSA (with the secp256k1 elliptic curve) and Ed25519. The hard work is the details of each of those steps.
+The hard work is the details of each of those steps.
 
 ***Notes: some useful links for signing:***
 
@@ -32,6 +31,8 @@ For example, the `Flags` [common transaction field](transaction-common-fields.ht
 
 All fields in a transaction are sorted in a specific order based on the field's type first, then the field itself second. (Think of it as sorting by family name, then given name, where the family name is the field's type and the given name is the field itself.)
 
+When you combine the type code and sort code, you get the field's identifier, which is prefixed before the field in the final serialized blob. If both the type code and the field code 
+
 ### Type Codes
 
 Each field type has an arbitrary sort code, with lower codes sorting first. These codes are defined in [`SField.h`](https://github.com/ripple/rippled/blob/master/src/ripple/protocol/SField.h#L57-L74).
@@ -40,7 +41,7 @@ For example, [UInt32 has sort order 2](https://github.com/ripple/rippled/blob/72
 
 ### Field Codes
 
-Each field also has a sort code, which is used to sort fields that have the same type as one another, with lower codes sorting first. These fields are defined in [`SField.cpp`](https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L72-L266).
+Each field also has a field code, which is used to sort fields that have the same type as one another, with lower codes sorting first. These fields are defined in [`SField.cpp`](https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L72-L266).
 
 For example, the `Account` field of a [Payment transaction][] [has sort code 1](https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L219), so it comes before the `Destination` field which [has sort code 3](https://github.com/ripple/rippled/blob/72e6005f562a8f0818bc94803d222ac9345e1e40/src/ripple/protocol/impl/SField.cpp#L221).
 
@@ -55,4 +56,14 @@ Some fields, such as `SignerEntry` (in [SignerListSet transactions][]), and `Mem
 
 ### Amount Fields
 
-The "AMOUNT" type is a special field type that represents an amount of currency, either XRP or an issued currency. ***TODO: details on how both are serialized in transactions.***
+The "Amount" type is a special field type that represents an amount of currency, either XRP or an issued currency. This type consists of two sub-types:
+
+- **XRP**
+    XRP is serialized as a 64-bit unsigned integer (big-endian order), except that the second-most-significant bit is `1` to indicate that it is positive. In other words, take a standard UInt64 and calculate the bitwise-OR of that with `0x4000000000000000` to get the serialized format.
+- **Issued Currencies**
+    Issued currencies consist of three segments in order:
+    1. 64 bits indicating the amount in the [internal currency format](currency-formats.html#issued-currency-math). The first bit is `1` to indicate that this is not XRP.
+    2. 160 bits indicating the [currency code](https://developers.ripple.com/currency-formats.html#currency-codes). The standard API converts 3-character codes such as "USD" into 160-bit codes using the [standard currency code format](currency-formats.html#standard-currency-codes), but custom 160-bit codes are also possible.
+    3. 160 bits indicating the issuer's Account ID. (See also: [Account Address Encoding](accounts.html#address-encoding))
+
+You can tell which of the two sub-types it is based on the first bit: `0` for XRP; `1` for issued currency.