From d810f5f9a567d8e82b10ada7ddbf7cf4c1145111 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 7 Mar 2012 09:14:00 -0800 Subject: [PATCH 1/7] Missing from previous commits. --- Application.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Application.h b/Application.h index 39cbb762ba..ee6431e374 100644 --- a/Application.h +++ b/Application.h @@ -8,6 +8,7 @@ #include "PubKeyCache.h" #include "ScopedLock.h" #include "LedgerMaster.h" +#include "LedgerAcquire.h" #include "TransactionMaster.h" #include "Wallet.h" #include "Peer.h" @@ -42,6 +43,7 @@ class Application KnownNodeList mKnownNodes; PubKeyCache mPKCache; LedgerMaster mMasterLedger; + LedgerAcquireMaster mMasterLedgerAcquire; TransactionMaster mMasterTransaction; DatabaseCon *mTxnDB, *mLedgerDB, *mWalletDB, *mHashNodeDB, *mNetNodeDB; @@ -72,6 +74,7 @@ public: boost::asio::io_service& getIOService() { return mIOService; } LedgerMaster& getMasterLedger() { return mMasterLedger; } + LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } DatabaseCon* getTxnDB() { return mTxnDB; } From efa2f90b34512a959de7dca0758e9c32e29714a9 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 7 Mar 2012 09:14:22 -0800 Subject: [PATCH 2/7] Cleanup. --- Conversion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Conversion.cpp b/Conversion.cpp index d151819103..9336867d1c 100644 --- a/Conversion.cpp +++ b/Conversion.cpp @@ -21,7 +21,7 @@ uint160 humanTo160(const std::string& buf) vector retVec; DecodeBase58(buf,retVec); uint160 ret; - memcpy((unsigned char*)&ret,&retVec[0],ret.GetSerializeSize()); + memcpy(reinterpret_cast(&ret), &retVec[0], ret.GetSerializeSize()); return(ret); From f08107cd039fd5ffe2418bf750fcfef7999e4377 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 7 Mar 2012 09:14:38 -0800 Subject: [PATCH 3/7] Punish a peer that sends us unwanted ledger data. --- Peer.cpp | 2 ++ Peer.h | 1 + 2 files changed, 3 insertions(+) diff --git a/Peer.cpp b/Peer.cpp index 6c50720642..8f48dd86b5 100644 --- a/Peer.cpp +++ b/Peer.cpp @@ -510,6 +510,8 @@ void Peer::recvGetLedger(newcoin::TMGetLedger& packet) void Peer::recvLedger(newcoin::TMLedgerData& packet) { + if(!theApp->getMasterLedgerAcquire().gotLedgerData(packet)) + punishPeer(PP_UNWANTED_DATA); } void Peer::sendHello() diff --git a/Peer.h b/Peer.h index f874bbd46d..0ddda232be 100644 --- a/Peer.h +++ b/Peer.h @@ -16,6 +16,7 @@ enum PeerPunish { PP_INVALID_REQUEST=1, // The peer sent a request that makes no sense PP_UNKNOWN_REQUEST=2, // The peer sent a request that might be garbage + PP_UNWANTED_DATA=3, // The peer sent us data we didn't want/need }; class Peer : public boost::enable_shared_from_this From d4107534bfc3606887d6169d553e4732a9015ac5 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 7 Mar 2012 09:14:50 -0800 Subject: [PATCH 4/7] Slots for the new transaction type handlers. --- Serializer.h | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/Serializer.h b/Serializer.h index 9e20ac1b9f..59ff3f8612 100644 --- a/Serializer.h +++ b/Serializer.h @@ -3,12 +3,15 @@ #include #include +#include #include #include "key.h" #include "uint256.h" +typedef std::pair > TaggedListItem; + class Serializer { public: @@ -32,6 +35,10 @@ class Serializer int addRaw(const std::vector &vector); int addRaw(const void *ptr, int len); + int addVL(const std::vector &vector); + int addVL(const void *ptr, int len); + int addTaggedList(const std::list&); + // disassemble functions bool get8(int&, int offset) const; bool get8(unsigned char&, int offset) const; @@ -41,9 +48,11 @@ class Serializer bool get160(uint160&, int offset) const; bool get256(uint256&, int offset) const; uint256 get256(int offset) const; + bool getVLLength(int& length, int offset) const; + bool getTaggedList(std::list&, int offset) const; bool getRaw(std::vector&, int offset, int length) const; std::vector getRaw(int offset, int length) const; - + // hash functions uint160 getRIPEMD160(int size=-1) const; uint256 getSHA256(int size=-1) const; @@ -69,7 +78,38 @@ class Serializer bool makeSignature(std::vector& signature, CKey& rkey) const; bool addSignature(CKey& rkey); + // VL length encode/decode functions + static std::vector encodeVL(int length) throw(); + static int getVLLength(int b1) throw(); + static int decodeVLLength(const std::vector&, int offset=0); + static void TestSerializer(); }; +class SerializerIterator +{ +protected: + const Serializer& mSerializer; + int mPos; + +public: + SerializerIterator(const Serializer& s) : mSerializer(s), mPos(0) { ; } + + void reset(void) { mPos=0; } + void setPos(int p) { mPos = p; } + const Serializer& operator*(void) { return mSerializer; } + + int getPos(void) { return mPos; } + int getBytesLeft(); + + unsigned char get8() throw(); + uint16 get16() throw(); + uint32 get32() throw(); + uint64 get64() throw(); + uint160 get160() throw(); + uint256 get256() throw(); + std::vector getVL() throw(); + std::list getTaggedList() throw(); +}; + #endif From 543c1af546650833100118f5f292458a1f2b6e64 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 11 Mar 2012 17:25:17 -0700 Subject: [PATCH 5/7] Variable-length and tagged list serializer functions. --- src/Serializer.cpp | 175 +++++++++++++++++++++++++++++++++++++++++++++ src/Serializer.h | 17 +++-- 2 files changed, 187 insertions(+), 5 deletions(-) diff --git a/src/Serializer.cpp b/src/Serializer.cpp index 15a9bf6106..af5c36bc35 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -229,6 +229,181 @@ bool Serializer::addSignature(CKey& key) return true; } +int Serializer::addVL(const std::vector& vector) +{ + int ret=addRaw(encodeVL(vector.size())); + addRaw(vector); + return ret; +} + +int Serializer::addVL(const void *ptr, int len) +{ + int ret=addRaw(encodeVL(len)); + addRaw(ptr, len); + return ret; +} + +int Serializer::addTaggedList(const std::list& list) +{ + int size=list.size(); + if(size>255) return -1; + int ret=add8(size); + if(size!=0) + for(std::list::const_iterator it=list.begin(); it!=list.end(); ++it) + { + add8(it->first); + addVL(it->second); + } + return ret; +} + +bool Serializer::getVL(std::vector& objectVL, int offset, int& length) const +{ + int b1; + if(!get8(b1, offset++)) return false; + + int datLen, lenLen=decodeLengthLength(b1); + try + { + if(lenLen==1) + datLen=decodeVLLength(b1); + else if(lenLen==2) + { + int b2; + if(!get8(b2, offset++)) return false; + datLen=decodeVLLength(b1, b2); + } + else if(lenLen==3) + { + int b2, b3; + if(!get8(b2, offset++)) return false; + if(!get8(b3, offset++)) return false; + datLen=decodeVLLength(b1, b2, b3); + } + else return false; + } + catch(...) + { + return false; + } + length=lenLen+datLen; + return getRaw(objectVL, offset, datLen); +} + +bool Serializer::getVLLength(int& length, int offset) const +{ + int b1; + if(!get8(b1, offset++)) return false; + + int lenLen=decodeLengthLength(b1); + try + { + if(lenLen==1) + length=decodeVLLength(b1); + else if(lenLen==2) + { + int b2; + if(!get8(b2, offset++)) return false; + length=decodeVLLength(b1, b2); + } + else if(lenLen==3) + { + int b2, b3; + if(!get8(b2, offset++)) return false; + if(!get8(b3, offset++)) return false; + length=decodeVLLength(b1, b2, b3); + } + else return false; + } + catch(...) + { + return false; + } + return true; +} + +bool Serializer::getTaggedList(std::list& list, int offset) const +{ + list.clear(); + int numElem; + if(!get8(numElem, offset++)) return false; + for(int i=0; i data; + if(!get8(tag, offset++)) return false; + if(!getVL(data, offset, len)) return false; + offset+=len; + list.push_back(std::make_pair(tag, data)); + } + return true; +} + +std::vector Serializer::encodeVL(int length) throw() +{ + unsigned char lenBytes[4]; + if(length<=192) + { + lenBytes[0]=static_cast(length); + return std::vector(&lenBytes[0], &lenBytes[1]); + } + else if(length<=12480) + { + length-=193; + lenBytes[0]=static_cast(length>>8); + lenBytes[1]=static_cast(length&0xff); + return std::vector(&lenBytes[0], &lenBytes[2]); + } + else if(length<=918744) + { + length-=12481; + lenBytes[0]=static_cast(length>>16); + lenBytes[1]=static_cast((length>>8)&0xff); + lenBytes[2]=static_cast(length&0xff); + return std::vector(&lenBytes[0], &lenBytes[3]); + } + else throw(std::overflow_error("lenlen")); +} + +int encodeLengthLength(int length) throw() +{ + if(length<0) throw(std::overflow_error("len<0")); + if(length<=192) return 1; + if(length<=12480) return 2; + if(length>=918744) return 3; + throw(std::overflow_error("len>918644")); +} + +int Serializer::decodeLengthLength(int b1) throw() +{ + if(b1<0) throw(std::overflow_error("b1<0")); + if(b1<=192) return 1; + if(b1<=240) return 2; + if(b1<=254) return 3; + throw(std::overflow_error("b1>254")); +} + +int Serializer::decodeVLLength(int b1) throw() +{ + if(b1<0) throw(std::overflow_error("b1<0")); + if(b1>254) throw(std::overflow_error("b1>254")); + return b1; +} + +int Serializer::decodeVLLength(int b1, int b2) throw() +{ + if(b1<193) throw(std::overflow_error("b1<193")); + if(b1>240) throw(std::overflow_error("b1>240")); + return 193+(b1-193)*256+b2; +} + +int Serializer::decodeVLLength(int b1, int b2, int b3) throw() +{ + if(b1<241) throw(std::overflow_error("b1<241")); + if(b1>254) throw(std::overflow_error("b1>254")); + return 12481+(b1-241)*65536+b2*256+b3; +} + void Serializer::TestSerializer() { Serializer s(64); diff --git a/src/Serializer.h b/src/Serializer.h index 59ff3f8612..4797add74d 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -48,11 +48,14 @@ class Serializer bool get160(uint160&, int offset) const; bool get256(uint256&, int offset) const; uint256 get256(int offset) const; - bool getVLLength(int& length, int offset) const; - bool getTaggedList(std::list&, int offset) const; bool getRaw(std::vector&, int offset, int length) const; std::vector getRaw(int offset, int length) const; + bool getVL(std::vector& objectVL, int offset, int& length) const; + bool getVLLength(int& length, int offset) const; + bool getTaggedList(std::list&, int offset) const; + + // hash functions uint160 getRIPEMD160(int size=-1) const; uint256 getSHA256(int size=-1) const; @@ -78,10 +81,13 @@ class Serializer bool makeSignature(std::vector& signature, CKey& rkey) const; bool addSignature(CKey& rkey); - // VL length encode/decode functions + // low-level VL length encode/decode functions static std::vector encodeVL(int length) throw(); - static int getVLLength(int b1) throw(); - static int decodeVLLength(const std::vector&, int offset=0); + static int encodeLengthLength(int length) throw(); + static int decodeLengthLength(int b1) throw(); + static int decodeVLLength(int b1) throw(); + static int decodeVLLength(int b1, int b2) throw(); + static int decodeVLLength(int b1, int b2, int b3) throw(); static void TestSerializer(); }; @@ -108,6 +114,7 @@ public: uint64 get64() throw(); uint160 get160() throw(); uint256 get256() throw(); + std::vector getVL() throw(); std::list getTaggedList() throw(); }; From e8c1f656ee43ef530b714ca1f69f54faf4105815 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 11 Mar 2012 21:35:41 -0700 Subject: [PATCH 6/7] SerializerIterator implementation. --- src/Serializer.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++++- src/Serializer.h | 2 +- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/Serializer.cpp b/src/Serializer.cpp index af5c36bc35..1abec9cfe9 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -322,9 +322,10 @@ bool Serializer::getVLLength(int& length, int offset) const return true; } -bool Serializer::getTaggedList(std::list& list, int offset) const +bool Serializer::getTaggedList(std::list& list, int offset, int& length) const { list.clear(); + int startOffset=offset; int numElem; if(!get8(numElem, offset++)) return false; for(int i=0; i& list, int offset) cons offset+=len; list.push_back(std::make_pair(tag, data)); } + length=offset-startOffset; return true; } @@ -408,3 +410,74 @@ void Serializer::TestSerializer() { Serializer s(64); } + +int SerializerIterator::getBytesLeft() +{ + return mSerializer.getLength()-mPos; +} + +unsigned char SerializerIterator::get8() throw() +{ + int val; + if(!mSerializer.get8(val, mPos)) throw(0); + mPos++; + return val; +} + +uint16 SerializerIterator::get16() throw() +{ + uint16 val; + if(!mSerializer.get16(val, mPos)) throw(0); + mPos+=16/8; + return val; +} + +uint32 SerializerIterator::get32() throw() +{ + uint32 val; + if(!mSerializer.get32(val, mPos)) throw(0); + mPos+=32/8; + return val; +} + +uint64 SerializerIterator::get64() throw() +{ + uint64 val; + if(!mSerializer.get64(val, mPos)) throw(0); + mPos+=64/8; + return val; +} + +uint160 SerializerIterator::get160() throw() +{ + uint160 val; + if(!mSerializer.get160(val, mPos)) throw(0); + mPos+=160/8; + return val; +} + +uint256 SerializerIterator::get256() throw() +{ + uint256 val; + if(!mSerializer.get256(val, mPos)) throw(0); + mPos+=256/8; + return val; +} + +std::vector SerializerIterator::getVL() throw() +{ + int length; + std::vector vl; + if(!mSerializer.getVL(vl, mPos, length)) throw(0); + mPos+=length; + return vl; +} + +std::list SerializerIterator::getTaggedList() throw() +{ + int length; + std::list tl; + if(!mSerializer.getTaggedList(tl, mPos, length)) throw(0); + mPos+=length; + return tl; +} diff --git a/src/Serializer.h b/src/Serializer.h index 4797add74d..3f7efd5a1e 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -53,7 +53,7 @@ class Serializer bool getVL(std::vector& objectVL, int offset, int& length) const; bool getVLLength(int& length, int offset) const; - bool getTaggedList(std::list&, int offset) const; + bool getTaggedList(std::list&, int offset, int& legnth) const; // hash functions From 429af40139cc7251c77d9faa0a1d58b17bd55966 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 12 Mar 2012 15:25:56 -0700 Subject: [PATCH 7/7] Add headers for serialized objects and types. This is a table-driver serialized that can also serialized/deserialize to/from binary, text, and database formats. --- src/SerializedObject.h | 49 +++++++++ src/SerializedTypes.h | 242 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 src/SerializedObject.h create mode 100644 src/SerializedTypes.h diff --git a/src/SerializedObject.h b/src/SerializedObject.h new file mode 100644 index 0000000000..4c67ece7e2 --- /dev/null +++ b/src/SerializedObject.h @@ -0,0 +1,49 @@ +#ifndef __SERIALIZEDOBJECT__ +#define __SERIALIZEDOBJECT__ + +#include "SerializedTypes.h" + +struct SOElement +{ // An element in the description of a serialized object + const char *e_name; + int e_flag; + SerializedTypeID e_id; + + SOElement(const char *n, int f, SerializedTypeID i) : e_name(n), e_flag(f), e_id(i) { ; } +}; + +struct SOType +{ // A type of serialized object + const char *name; + std::list elements; +}; + +class STUObject : public SerializedType +{ +protected: + SOType *type; + std::list data; + +public: + STUObject() { ; } + virtual ~STUObject(); + + STUObject(const STUObject&); + STUObject& operator=(const STUObject&); + + int getLength() const; + SerializedTypeID getType() const { return STI_OBJECT; } + STUObject* duplicate() const { return new STUObject(*this); } + + std::vector serialize() const; + std::string getText() const; + std::string getSQL() const; + + void addObject(const SerializedType& t) { data.push_back(t.duplicate()); } + void giveObject(SerializedType* t) { data.push_back(t); } + const std::list& peekData() const { return data; } + std::list& peekData() { return data; } +}; + + +#endif diff --git a/src/SerializedTypes.h b/src/SerializedTypes.h new file mode 100644 index 0000000000..626513fb7b --- /dev/null +++ b/src/SerializedTypes.h @@ -0,0 +1,242 @@ +#ifndef __SERIALIZEDTYPES__ +#define __SERIALIZEDTYPES__ + +#include +#include + +#include "uint256.h" +#include "Serializer.h" + +enum SerializedTypeID +{ + STI_OBJECT=0, + STI_UINT8=1, STI_UINT16=2, STI_UINT32=4, STI_UINT64=5, + STI_HASH160=6, STI_HASH256=7, STI_VL=8, STI_TL=8 +}; + +class SerializedType +{ +public: + + SerializedType() { ; } + virtual ~SerializedType() { ; } + + virtual int getLength() const=0; + virtual SerializedTypeID getType() const=0; + virtual SerializedType* duplicate() const=0; + + virtual std::string getText() const=0; + virtual std::string getSQL() const=0; + + virtual std::vector serialize() const=0; + virtual int add(std::vector& j) const; +}; + +class STUInt8 : public SerializedType +{ +protected: + unsigned char value; + +public: + + STUInt8(unsigned char v=0) : value(v) { ; } + static STUInt8* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 1; } + SerializedTypeID getType() const { return STI_UINT8; } + STUInt8 *duplicate() const { return new STUInt8(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + unsigned char getValue() const { return value; } + void setValue(unsigned char v) { value=v; } + + operator unsigned char() const { return value; } + STUInt8& operator=(unsigned char v) { value=v; return *this; } +}; + +class STUInt16 : public SerializedType +{ +protected: + uint16 value; + +public: + + STUInt16(uint16 v=0) : value(v) { ; } + static STUInt16* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 2; } + SerializedTypeID getType() const { return STI_UINT16; } + STUInt16 *duplicate() const { return new STUInt16(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + uint16 getValue() const { return value; } + void setValue(uint16 v) { value=v; } + + operator uint16() const { return value; } + STUInt16& operator=(uint16 v) { value=v; return *this; } +}; + +class STUInt32 : public SerializedType +{ +protected: + uint32 value; + +public: + + STUInt32(uint32 v=0) : value(v) { ; } + static STUInt32* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 4; } + SerializedTypeID getType() const { return STI_UINT32; } + STUInt32 *duplicate() const { return new STUInt32(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + uint32 getValue() const { return value; } + void setValue(uint32 v) { value=v; } + + operator uint32() const { return value; } + STUInt32& operator=(uint32 v) { value=v; return *this; } +}; + +class STUInt64 : public SerializedType +{ +protected: + uint64 value; + +public: + + STUInt64(uint64 v=0) : value(v) { ; } + static STUInt64* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 8; } + SerializedTypeID getType() const { return STI_UINT64; } + STUInt64 *duplicate() const { return new STUInt64(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + uint64 getValue() const { return value; } + void setValue(uint64 v) { value=v; } + + operator uint64() const { return value; } + STUInt64& operator=(uint64 v) { value=v; return *this; } +}; + +class STUHash160 : public SerializedType +{ +protected: + uint160 value; + +public: + + STUHash160(const uint160& v) : value(v) { ; } + STUHash160() { ; } + static STUHash160* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 20; } + SerializedTypeID getType() const { return STI_HASH160; } + STUHash160 *duplicate() const { return new STUHash160(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + const uint160& getValue() const { return value; } + void setValue(const uint160& v) { value=v; } + + operator uint160() const { return value; } + STUHash160& operator=(const uint160& v) { value=v; return *this; } +}; + +class STUHash256 : public SerializedType +{ +protected: + uint256 value; + +public: + + STUHash256(const uint256& v) : value(v) { ; } + STUHash256() { ; } + static STUHash256* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const { return 32; } + SerializedTypeID getType() const { return STI_HASH256; } + STUHash256 *duplicate() const { return new STUHash256(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + const uint256& getValue() const { return value; } + void setValue(const uint256& v) { value=v; } + + operator uint256() const { return value; } + STUHash256& operator=(const uint256& v) { value=v; return *this; } +}; + +class STUVariableLength +{ // variable length byte string protected: +protected: + std::vector value; + +public: + + STUVariableLength(const std::vector& v) : value(v) { ; } + STUVariableLength() { ; } + static STUVariableLength* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const; + SerializedTypeID getType() const { return STI_VL; } + STUVariableLength *duplicate() const { return new STUVariableLength(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + const std::vector& peekValue() const { return value; } + std::vector& peekValue() { return value; } + std::vector getValue() const { return value; } + void setValue(std::vector& v) { value=v; } + + operator std::vector() const { return value; } + STUVariableLength& operator=(const std::vector& v) { value=v; return *this; } +}; + +class STUTaggedList +{ +protected: + std::list value; + +public: + + STUTaggedList(const std::list& v) : value(v) { ; } + STUTaggedList() { ; } + static STUTaggedList* construct(const std::vector&, int start_offset, int& length_consumed); + + int getLength() const; + SerializedTypeID getType() const { return STI_TL; } + STUTaggedList *duplicate() const { return new STUTaggedList(value); } + std::string getText() const; + std::string getSQL() const; + std::vector serialize() const; + + const std::list& peekValue() const { return value; } + std::list& peekValue() { return value; } + std::list getValue() const { return value; } + + void setValue(std::list& v) { value=v; } + + int getItemCount() const { return value.size(); } + bool isEmpty() const { return value.empty(); } + + void clear() { value.erase(value.begin(), value.end()); } + void addItem(const TaggedListItem& v) { value.push_back(v); } + + operator std::list() const { return value; } + STUTaggedList& operator=(const std::list& v) { value=v; return *this; } +}; + +#endif