Serialization - sample code in progress, clarifications

content/_code-samples/tx-serialization/address.py

@@ -0,0 +1,8 @@
+import base58.base58 as base58
+
+def decode_address(address):
+    decoded = base58.b58decode_check(address)
+    if decoded[0] == 0 and len(decoded) == 21: # is an address
+        return decoded[1:]
+    else:
+        raise ValueError("Not an AccountID!")

content/_code-samples/tx-serialization/base58/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2015 David Keijser
+
+	Permission is hereby granted, free of charge, to any person obtaining a copy
+	of this software and associated documentation files (the "Software"), to deal
+	in the Software without restriction, including without limitation the rights
+	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+	copies of the Software, and to permit persons to whom the Software is
+	furnished to do so, subject to the following conditions:
+
+	The above copyright notice and this permission notice shall be included in
+	all copies or substantial portions of the Software.
+
+	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+	THE SOFTWARE.

content/_code-samples/tx-serialization/base58/base58.py

@@ -0,0 +1,177 @@
+'''Base58 encoding
+
+Implementations of Base58 and Base58Check endcodings that are compatible
+with the XRP Ledger.
+'''
+
+# This This code is adapted from the module by David Keijser at
+# <https://github.com/keis/base58>. - rome@ripple.com
+# His notes are preserved below:
+
+# This module is based upon base58 snippets found scattered over many bitcoin
+# tools written in python. From what I gather the original source is from a
+# forum post by Gavin Andresen, so direct your praise to him.
+# This module adds shiny packaging and support for python3.
+
+from hashlib import sha256
+
+__version__ = '1.0.3-xrp'
+
+# 58 character alphabet used
+# alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' # Bitcoin
+alphabet = b'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz' # XRP Ledger
+
+
+if bytes == str:  # python2
+    iseq, bseq, buffer = (
+        lambda s: map(ord, s),
+        lambda s: ''.join(map(chr, s)),
+        lambda s: s,
+    )
+else:  # python3
+    iseq, bseq, buffer = (
+        lambda s: s,
+        bytes,
+        lambda s: s.buffer,
+    )
+
+
+def scrub_input(v):
+    if isinstance(v, str) and not isinstance(v, bytes):
+        v = v.encode('ascii')
+
+    if not isinstance(v, bytes):
+        raise TypeError(
+            "a bytes-like object is required (also str), not '%s'" %
+            type(v).__name__)
+
+    return v
+
+
+def b58encode_int(i, default_one=True):
+    '''Encode an integer using Base58'''
+    if not i and default_one:
+        return alphabet[0:1]
+    string = b""
+    while i:
+        i, idx = divmod(i, 58)
+        string = alphabet[idx:idx+1] + string
+    return string
+
+
+def b58encode(v):
+    '''Encode a string using Base58'''
+
+    v = scrub_input(v)
+
+    nPad = len(v)
+    v = v.lstrip(b'\0')
+    nPad -= len(v)
+
+    p, acc = 1, 0
+    for c in iseq(reversed(v)):
+        acc += p * c
+        p = p << 8
+
+    result = b58encode_int(acc, default_one=False)
+
+    return (alphabet[0:1] * nPad + result)
+
+
+def b58decode_int(v):
+    '''Decode a Base58 encoded string as an integer'''
+
+    v = scrub_input(v)
+
+    decimal = 0
+    for char in v:
+        decimal = decimal * 58 + alphabet.index(char)
+    return decimal
+
+
+def b58decode(v):
+    '''Decode a Base58 encoded string'''
+
+    v = scrub_input(v)
+
+    origlen = len(v)
+    v = v.lstrip(alphabet[0:1])
+    newlen = len(v)
+
+    acc = b58decode_int(v)
+
+    result = []
+    while acc > 0:
+        acc, mod = divmod(acc, 256)
+        result.append(mod)
+
+    return (b'\0' * (origlen - newlen) + bseq(reversed(result)))
+
+
+def b58encode_check(v):
+    '''Encode a string using Base58 with a 4 character checksum'''
+
+    digest = sha256(sha256(v).digest()).digest()
+    return b58encode(v + digest[:4])
+
+
+def b58decode_check(v):
+    '''Decode and verify the checksum of a Base58 encoded string'''
+
+    result = b58decode(v)
+    result, check = result[:-4], result[-4:]
+    digest = sha256(sha256(result).digest()).digest()
+
+    if check != digest[:4]:
+        raise ValueError("Invalid checksum")
+
+    return result
+
+
+def main():
+    '''Base58 encode or decode FILE, or standard input, to standard output.'''
+
+    import sys
+    import argparse
+
+    stdout = buffer(sys.stdout)
+
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument(
+        'file',
+        metavar='FILE',
+        nargs='?',
+        type=argparse.FileType('r'),
+        default='-')
+    parser.add_argument(
+        '-d', '--decode',
+        action='store_true',
+        help='decode data')
+    parser.add_argument(
+        '-c', '--check',
+        action='store_true',
+        help='append a checksum before encoding')
+
+    args = parser.parse_args()
+    fun = {
+        (False, False): b58encode,
+        (False, True): b58encode_check,
+        (True, False): b58decode,
+        (True, True): b58decode_check
+    }[(args.decode, args.check)]
+
+    data = buffer(args.file).read()
+
+    try:
+        result = fun(data)
+    except Exception as e:
+        sys.exit(e)
+
+    if not isinstance(result, bytes):
+        result = result.encode('ascii')
+
+    stdout.write(result)
+
+
+if __name__ == '__main__':
+    main()

content/_code-samples/tx-serialization/field_ordering.py

@@ -0,0 +1,180 @@
+# Canonical ordering (of transaction signing fields only)
+# for serializing and signing transactions
+
+TYPES = {
+    # Type name: canonical order for this type
+    "UInt16": 1,
+    "UInt32": 2,
+    "UInt64": 3,
+    "Hash128": 4,
+    "Hash256": 5,
+    "Amount": 6,
+    "VL": 7, # VariableLength
+    "Account": 8,
+    # 9-13 are reserved
+    "Object": 14,
+    "Array": 15,
+    "UInt8": 16,
+    "Hash160": 17,
+    "PathSet": 18, # "Paths" field of cross-currency payments only
+    "Vector256": 19,
+}
+
+FIELDS = {
+    # Type name: Canonical order for this field (among fields of its type)
+
+    # UInt16 types
+    "LedgerEntryType": 1,
+    "TransactionType": 2,
+    "SignerWeight": 3,
+    # UInt32 types
+    "Flags": 2,
+    "SourceTag": 3,
+    "Sequence": 4,
+    "PreviousTxnLgrSeq": 5,
+    "LedgerSequence": 6,
+    "CloseTime": 7,
+    "ParentCloseTime": 8,
+    "SigningTime": 9,
+    "Expiration": 10,
+    "TransferRate": 11,
+    "WalletSize": 12,
+    "OwnerCount": 13,
+    "DestinationTag": 14,
+    "HighQualityIn": 16,
+    "HighQualityOut": 17,
+    "LowQualityIn": 18,
+    "LowQualityOut": 19,
+    "QualityIn": 20,
+    "QualityOut": 21,
+    "StampEscrow": 22,
+    "BondAmount": 23,
+    "LoadFee": 24,
+    "OfferSequence": 25,
+    "FirstLedgerSequence": 26,
+    "LastLedgerSequence": 27,
+    "TransactionIndex": 28,
+    "OperationLimit": 29,
+    "ReferenceFeeUnits": 30,
+    "ReserveBase": 31,
+    "ReserveIncrement": 32,
+    "SetFlag": 33,
+    "ClearFlag": 34,
+    "SignerQuorum": 35,
+    "CancelAfter": 36,
+    "FinishAfter": 37,
+    "SignerListID": 38,
+    "SettleDelay": 39,
+    # UInt64 types
+    "IndexNext": 1,
+    "IndexPrevious": 2,
+    "BookNode": 3,
+    "OwnerNode": 4,
+    "BaseFee": 5,
+    "ExchangeRate": 6,
+    "LowNode": 7,
+    "HighNode": 8,
+    "DestinationNode": 9,
+    "Cookie": 10,
+    # Hash128 types
+    "EmailHash": 1,
+    # Hash256 types
+    "LedgerHash": 1,
+    "ParentHash": 2,
+    "TransactionHash": 3,
+    "AccountHash": 4,
+    "PreviousTxnID": 5,
+    "LedgerIndex": 6,
+    "WalletLocator": 7,
+    "RootIndex": 8,
+    "AccountTxnID": 9,
+    "BookDirectory": 16,
+    "InvoiceID": 17,
+    "Nickname": 18,
+    "Amendment": 19,
+    "TicketID": 20,
+    "Digest": 21,
+    "Channel": 22,
+    "ConsensusHash": 23,
+    "CheckID": 24,
+    "hash": 257,
+    "index": 258,
+    # Amount types
+    "Amount": 1,
+    "Balance": 2,
+    "LimitAmount": 3,
+    "TakerPays": 4,
+    "TakerGets": 5,
+    "LowLimit": 6,
+    "HighLimit": 7,
+    "Fee": 8,
+    "SendMax": 9,
+    "DeliverMin": 10,
+    "MinimumOffer": 16,
+    "RippleEscrow": 17,
+    "DeliveredAmount": 18,
+    # VL types
+    "PublicKey": 1,
+    "MessageKey": 2,
+    "SigningPubKey": 3,
+    "TxnSignature": 4,
+    "Signature": 6,
+    "Domain": 7,
+    "FundCode": 8,
+    "RemoveCode": 9,
+    "ExpireCode": 10,
+    "CreateCode": 11,
+    "MemoType": 12,
+    "MemoData": 13,
+    "MemoFormat": 14,
+    "Fulfillment": 16,
+    "Condition": 17,
+    "MasterSignature": 18,
+    # Account types
+    "Account": 1,
+    "Owner": 2,
+    "Destination": 3,
+    "Issuer": 4,
+    "Authorize": 5,
+    "Unauthorize": 6,
+    "Target": 7,
+    "RegularKey": 8,
+    # Object types
+    "TransactionMetaData": 2,
+    "CreatedNode": 3,
+    "DeletedNode": 4,
+    "ModifiedNode": 5,
+    "PreviousFields": 6,
+    "FinalFields": 7,
+    "NewFields": 8,
+    "TemplateEntry": 9,
+    "Memo": 10,
+    "SignerEntry": 11,
+    "Signer": 16,
+    "Majority": 18,
+    # Array types
+    "Signers": 3,
+    "SignerEntries": 4,
+    "Template": 5,
+    "Necessary": 6,
+    "Sufficient": 7,
+    "AffectedNodes": 8,
+    "Memos": 9,
+    "Majorities": 16,
+    # UInt8 types
+    "CloseResolution": 1,
+    "Method": 2,
+    "TransactionResult": 3,
+    "TickSize": 16,
+    # Hash160 types
+    "TakerPaysCurrency": 1,
+    "TakerPaysIssuer": 2,
+    "TakerGetsCurrency": 3,
+    "TakerGetsIssuer": 4,
+    # PathSet types
+    "Paths": 1,
+    # Vector256 types
+    "Indexes": 1,
+    "Hashes": 2,
+    "Amendments": 3,
+}

content/_code-samples/tx-serialization/serialize.py

@@ -0,0 +1,154 @@
+#!/bin/env python3
+
+# Transaction Serialization Sample Code (Python3 version)
+# Author: rome@ripple.com
+# Copyright Ripple 2018
+
+import json
+import logging
+
+from address import decode_address
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+
+def load_defs(fname="definitions.json"):
+    with open(fname) as definitions_file:
+        definitions = json.load(definitions_file)
+        return {
+            "TYPES": definitions["TYPES"],
+                # type_name str: type_sort_key int
+            "FIELDS": {k:v for (k,v) in definitions["FIELDS"]}, # convert list of tuples to dict
+                # field_name str: {
+                #   nth: field_sort_key int,
+                #   isVLEncoded: bool,
+                #   isSerialized: bool,
+                #   isSigningField: bool,
+                #   type: type_name str
+                # }
+            "LEDGER_ENTRY_TYPES": definitions["LEDGER_ENTRY_TYPES"],
+            "TRANSACTION_RESULTS": definitions["TRANSACTION_RESULTS"],
+            "TRANSACTION_TYPES": definitions["TRANSACTION_TYPES"],
+        }
+
+
+def field_sort_key(field_name):
+    """Return a tuple sort key for a given field name"""
+    field_type_name = DEFINITIONS["FIELDS"][field_name]["type"]
+    return (DEFINITIONS["TYPES"][field_type_name], DEFINITIONS["FIELDS"][field_name]["nth"])
+
+def field_id(field_name):
+    field_type_name = DEFINITIONS["FIELDS"][field_name]["type"]
+    type_code = DEFINITIONS["TYPES"][field_type_name]
+    field_code = DEFINITIONS["FIELDS"][field_name]["nth"]
+
+    if type_code < 16 and field_code < 16:
+        # first 4 bits is the type_code, next 4 bits is the field code
+        combined_code = (type_code << 4) | field_code
+        return combined_code.to_bytes(1, byteorder="big", signed=False)
+    else:
+        # TODO: need more than 1 byte to encode this field id
+        raise ValueError("field_id not yet implemented for types/fields > 16")
+
+def bytes_from_uint(i, bits):
+    if bits % 8:
+        raise ValueError("bytes_from_uint: bits must be divisible by 8")
+    return i.to_bytes(bits // 8, byteorder="big", signed=False)
+
+def amount_to_bytes(a):
+    if type(a) == str:
+        # is XRP
+        xrp_amt = int(a)
+        if (xrp_amt >= 0):
+            # set the "is positive" bit -- this is backwards from usual two's complement!
+            xrp_amt = xrp_amt | 0x4000000000000000
+        else:
+            # convert to absolute value, leaving the "is positive" bit unset
+            xrp_amt = -xrp_amt
+        return xrp_amt.to_bytes(8, byteorder="big", signed=False)
+    elif type(a) == dict:
+        if sorted(a.keys()) != ["currency", "issuer", "value"]:
+            raise ValueError("amount must have currency, value, issuer only (actually had: %s)" %
+                    sorted(a.keys()))
+        #TODO: canonicalize mantissa/exponent/etc. of issued currency amount
+        # https://developers.ripple.com/currency-formats.html#issued-currency-math
+        # temporarily returning all zeroes
+        issued_amt = bytes(8)
+        # TODO: calculate 160-bit currency ID
+        currency_code = bytes(20)
+        return issued_amt + currency_code + decode_address(a["issuer"])
+    else:
+        raise ValueError("amount must be XRP string or {currency, value, issuer}")
+
+def tx_type_to_bytes(txtype):
+    type_uint = DEFINITIONS["TRANSACTION_TYPES"][txtype]
+    return type_uint.to_bytes(2, byteorder="big", signed=False)
+
+def field_to_bytes(field_name, field_val):
+    field_type = DEFINITIONS["FIELDS"][field_name]["type"]
+    logger.debug("Serializing field {f} of type {t}".format(f=field_name, t=field_type))
+
+    id_prefix = field_id(field_name)
+    logger.debug("id_prefix is: %s" % id_prefix.hex())
+
+    if field_name == "TransactionType":
+        # Special case: convert from string to UInt16
+        return b''.join( (id_prefix, tx_type_to_bytes(field_val)) )
+
+    dispatch = {
+        # TypeName: function(field): bytes object
+        "UInt64": lambda x:bytes_from_uint(x, 64),
+        "UInt32": lambda x:bytes_from_uint(x, 32),
+        "UInt16": lambda x:bytes_from_uint(x, 16),
+        "UInt8" : lambda x:bytes_from_uint(x, 8),
+        "AccountID": decode_address,
+        "Amount": amount_to_bytes
+    }
+    field_binary = dispatch[field_type](field_val)
+    return b''.join( (id_prefix, field_binary) )
+
+def serialize_tx(tx):
+    field_order = sorted(tx.keys(), key=field_sort_key)
+    logger.debug("Canonical field order: %s" % field_order)
+
+    fields_as_bytes = []
+    for field_name in field_order:
+        field_val = tx[field_name]
+        fields_as_bytes.append(field_to_bytes(field_name, field_val))
+
+    all_serial = b''.join(fields_as_bytes)
+    logger.info(all_serial.hex().upper())
+    return all_serial
+
+
+
+################################################################################
+
+if __name__ == "__main__":
+    logger.setLevel(logging.DEBUG)
+    DEFINITIONS = load_defs()
+
+    example_tx = {
+      "TransactionType" : "Payment",
+      "Account" : "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
+      "Destination" : "ra5nK24KXen9AHvsdFTKHSANinZseWnPcX",
+      "Amount" : {
+         "currency" : "USD",
+         "value" : "1",
+         "issuer" : "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn"
+      },
+      "Fee": "12",
+      "Flags": 2147483648,
+      "Sequence": 2
+    }
+
+    serialize_tx(example_tx)
+
+    # example rippled signature:
+    # ./rippled sign masterpassphrase (the above JSON)
+    # where "masterpassphrase" is the key behind rHb9...
+    # snoPBrXtMeMyMHUVTgbuqAfg1SUTb in base58
+    # "tx_blob" : "1200002280000000240000000261D4838D7EA4C6800000000000000000000000000055534400000000004B4E9C06F24296074F7BC48F92A97916C6DC5EA968400000000000000C73210330E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD0207446304402201FE0A74FC1BDB509C8F42B861EF747C43B92917706BB623F0A0D621891933AF402205206FBA8B0BF6733DB5B03AD76B5A76A2D46DF9093916A3BEC78897E58A3DF148114B5F762798A53D543A014CAF8B297CFF8F2F937E883143E9D4A2B8AA0780F682D136F7A56D6724EF53754",
+    # "SigningPubKey" : "0330E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD020",
+    # "TxnSignature" : "304402201FE0A74FC1BDB509C8F42B861EF747C43B92917706BB623F0A0D621891933AF402205206FBA8B0BF6733DB5B03AD76B5A76A2D46DF9093916A3BEC78897E58A3DF14",
+    # "hash" : "8BA1509E4FB80CCF76CD9DE924B8B71597637C775BA2DC515F90C333DA534BF3"