mirror of
https://github.com/XRPLF/clio.git
synced 2025-11-04 20:05:51 +00:00
feat: Snapshot exporting tool (#1877)
In this PR: 1 We create a golang grpc client to request data from rippled 2 We store the data into the specific place 3 Add unittests 4 Create build script, the build can be initiated by set conan option `snapshot` being true. Please ignore the grpc server part. It will be implemented in importing tool.
This commit is contained in:
cyan317
@@ -16,6 +16,8 @@ option(coverage "Build test coverage report" FALSE)
|
||||
option(packaging "Create distribution packages" FALSE)
|
||||
option(lint "Run clang-tidy checks during compilation" FALSE)
|
||||
option(static "Statically linked Clio" FALSE)
|
||||
option(snapshot "Build snapshot tool" FALSE)
|
||||
|
||||
# ========================================================================== #
|
||||
set(san "" CACHE STRING "Add sanitizer instrumentation")
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
|
||||
@@ -91,3 +93,7 @@ include(install/install)
|
||||
if (packaging)
|
||||
include(cmake/packaging.cmake) # This file exists only in build runner
|
||||
endif ()
|
||||
|
||||
if (snapshot)
|
||||
add_subdirectory(tools/snapshot)
|
||||
endif ()
|
||||
|
||||
@@ -19,6 +19,7 @@ class Clio(ConanFile):
|
||||
'packaging': [True, False], # create distribution packages
|
||||
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
|
||||
'lint': [True, False], # run clang-tidy checks during compilation
|
||||
'snapshot': [True, False], # build export/import snapshot tool
|
||||
}
|
||||
|
||||
requires = [
|
||||
@@ -44,6 +45,7 @@ class Clio(ConanFile):
|
||||
'coverage': False,
|
||||
'lint': False,
|
||||
'docs': False,
|
||||
'snapshot': False,
|
||||
|
||||
'xrpl/*:tests': False,
|
||||
'xrpl/*:rocksdb': False,
|
||||
@@ -92,6 +94,7 @@ class Clio(ConanFile):
|
||||
tc.variables['docs'] = self.options.docs
|
||||
tc.variables['packaging'] = self.options.packaging
|
||||
tc.variables['benchmark'] = self.options.benchmark
|
||||
tc.variables['snapshot'] = self.options.snapshot
|
||||
tc.generate()
|
||||
|
||||
def build(self):
|
||||
|
||||
66
tools/snapshot/CMakeLists.txt
Normal file
66
tools/snapshot/CMakeLists.txt
Normal file
@@ -0,0 +1,66 @@
|
||||
# Build snapshot tool
|
||||
# Need to be installed before building: go protoc-gen-go protoc-gen-go-grpc
|
||||
|
||||
set(GO_EXECUTABLE "go")
|
||||
set(GO_SOURCE_DIR "${CMAKE_SOURCE_DIR}/tools/snapshot")
|
||||
set(PROTO_INC_DIR "${xrpl_PACKAGE_FOLDER_RELEASE}/include/xrpl/proto")
|
||||
set(PROTO_SOURCE_DIR "${PROTO_INC_DIR}/org/xrpl/rpc/v1/")
|
||||
set(GO_OUTPUT "${CMAKE_BINARY_DIR}/clio_snapshot")
|
||||
|
||||
set(PROTO_FILES
|
||||
${PROTO_SOURCE_DIR}/xrp_ledger.proto ${PROTO_SOURCE_DIR}/ledger.proto ${PROTO_SOURCE_DIR}/get_ledger.proto
|
||||
${PROTO_SOURCE_DIR}/get_ledger_entry.proto ${PROTO_SOURCE_DIR}/get_ledger_data.proto
|
||||
${PROTO_SOURCE_DIR}/get_ledger_diff.proto
|
||||
)
|
||||
|
||||
execute_process(COMMAND go env GOPATH OUTPUT_VARIABLE GOPATH_VALUE OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
|
||||
# Target Go package path
|
||||
set(GO_IMPORT_PATH "org/xrpl/rpc/v1")
|
||||
|
||||
# Initialize the options strings
|
||||
set(GO_OPTS "")
|
||||
set(GRPC_OPTS "")
|
||||
|
||||
# Loop through each proto file
|
||||
foreach(proto ${PROTO_FILES})
|
||||
get_filename_component(proto_filename ${proto} NAME)
|
||||
|
||||
# Build the --go_opt and --go-grpc_opt mappings
|
||||
set(GO_OPTS ${GO_OPTS} --go_opt=M${GO_IMPORT_PATH}/${proto_filename}=${GO_IMPORT_PATH})
|
||||
set(GRPC_OPTS ${GRPC_OPTS} --go-grpc_opt=M${GO_IMPORT_PATH}/${proto_filename}=${GO_IMPORT_PATH})
|
||||
endforeach()
|
||||
|
||||
foreach (proto ${PROTO_FILES})
|
||||
get_filename_component(proto_name ${proto} NAME_WE)
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_SOURCE_DIR}/${proto_name}.pb.go
|
||||
COMMAND
|
||||
protoc ${GO_OPTS} ${GRPC_OPTS}
|
||||
--go-grpc_out=${GO_SOURCE_DIR} -I${PROTO_INC_DIR} ${proto} --plugin=${GOPATH_VALUE}/bin/protoc-gen-go
|
||||
--plugin=${GOPATH_VALUE}/bin/protoc-gen-go-grpc --go_out=${GO_SOURCE_DIR}/
|
||||
DEPENDS ${proto}
|
||||
COMMENT "Generating Go code for ${proto}"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${proto_name}.pb.go)
|
||||
endforeach ()
|
||||
|
||||
add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
|
||||
|
||||
add_custom_target(run_go_tests
|
||||
COMMAND go test ./...
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
COMMENT "Running clio_snapshot unittests"
|
||||
VERBATIM
|
||||
DEPENDS ${GENERATED_GO_FILES}
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_OUTPUT}
|
||||
COMMAND ${GO_EXECUTABLE} build -o ${GO_OUTPUT} ${GO_SOURCE_DIR}
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
COMMENT "Building clio_snapshot"
|
||||
VERBATIM
|
||||
)
|
||||
23
tools/snapshot/go.mod
Normal file
23
tools/snapshot/go.mod
Normal file
@@ -0,0 +1,23 @@
|
||||
module xrplf/clio/clio_snapshot
|
||||
|
||||
go 1.22
|
||||
|
||||
toolchain go1.22.11
|
||||
|
||||
require (
|
||||
github.com/golang/mock v1.6.0
|
||||
github.com/spf13/pflag v1.0.6
|
||||
github.com/stretchr/testify v1.10.0
|
||||
google.golang.org/grpc v1.69.4
|
||||
google.golang.org/protobuf v1.36.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
69
tools/snapshot/go.sum
Normal file
69
tools/snapshot/go.sum
Normal file
@@ -0,0 +1,69 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
|
||||
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
|
||||
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
|
||||
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
|
||||
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
|
||||
go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
|
||||
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
|
||||
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
|
||||
google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A=
|
||||
google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
|
||||
google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU=
|
||||
google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
245
tools/snapshot/internal/export/export.go
Normal file
245
tools/snapshot/internal/export/export.go
Normal file
@@ -0,0 +1,245 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
deltaDataFolderDiv = 10000
|
||||
grpcUser = "clio-snapshot"
|
||||
markerNum = 16
|
||||
)
|
||||
|
||||
type gRPCClient struct {
|
||||
Client pb.XRPLedgerAPIServiceClient
|
||||
conn *grpc.ClientConn
|
||||
}
|
||||
|
||||
func (c *gRPCClient) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
func createGRPCClient(serverAddr string) (*gRPCClient, error) {
|
||||
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
|
||||
|
||||
conn, err := grpc.NewClient(serverAddr, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to dial: %v", err)
|
||||
}
|
||||
|
||||
client := pb.NewXRPLedgerAPIServiceClient(conn)
|
||||
return &gRPCClient{
|
||||
Client: client,
|
||||
conn: conn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
request := pb.GetLedgerRequest{}
|
||||
ledger := &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: seq,
|
||||
},
|
||||
}
|
||||
request.Ledger = ledger
|
||||
request.User = grpcUser
|
||||
request.GetObjectNeighbors = true
|
||||
request.Transactions = true
|
||||
request.Expand = true
|
||||
request.GetObjects = true
|
||||
|
||||
response, err := client.GetLedger(ctx, &request)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
}
|
||||
|
||||
saveLedgerDeltaData(seq, response, path)
|
||||
|
||||
log.Printf("Processing delta sequence: %d\n", seq)
|
||||
}
|
||||
|
||||
func roundDown(n uint32, roundTo uint32) uint32 {
|
||||
if roundTo == 0 {
|
||||
return n
|
||||
}
|
||||
return n - (n % roundTo)
|
||||
}
|
||||
|
||||
func saveLedgerDeltaData(seq uint32, response *pb.GetLedgerResponse, path string) {
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
protoData, err := proto.Marshal(response)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
|
||||
|
||||
err = os.WriteFile(filePath, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func generateMarkers(markerNum uint32) [][32]byte {
|
||||
var byteArray [32]byte
|
||||
|
||||
incr := 256 / markerNum
|
||||
|
||||
var byteArrayList [][32]byte
|
||||
|
||||
for i := 0; i < int(markerNum); i++ {
|
||||
byteArray[0] = byte(i * int(incr)) // Increment the highest byte
|
||||
byteArrayList = append(byteArrayList, byteArray)
|
||||
}
|
||||
|
||||
return byteArrayList
|
||||
}
|
||||
|
||||
func saveLedgerData(path string, data *pb.GetLedgerDataResponse) {
|
||||
protoData, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, path string) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
request := pb.GetLedgerDataRequest{}
|
||||
ledger := &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: seq,
|
||||
},
|
||||
}
|
||||
request.Ledger = ledger
|
||||
request.Marker = marker[:]
|
||||
if end != nil {
|
||||
request.EndMarker = end[:]
|
||||
}
|
||||
request.User = grpcUser
|
||||
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%x", marker))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
for request.Marker != nil {
|
||||
res, err := client.GetLedgerData(ctx, &request)
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%x.dat", request.Marker))
|
||||
saveLedgerData(filePath, res)
|
||||
request.Marker = res.Marker
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
log.Printf("Processing full sequence: %d\n", seq)
|
||||
|
||||
markers := generateMarkers(markerNum)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Launch a goroutine for each marker
|
||||
for i, marker := range markers {
|
||||
wg.Add(1)
|
||||
var end []byte
|
||||
if i != len(markers)-1 {
|
||||
end = markers[i+1][:]
|
||||
}
|
||||
|
||||
fmt.Printf("Got ledger data marker: %x-%x\n", marker, end)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
getLedgerData(client, seq, marker[:], end, path)
|
||||
}()
|
||||
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func checkPath(path string) {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
// Create the directory if it doesn't exist
|
||||
err := os.MkdirAll(path, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
}
|
||||
|
||||
defer client.Close()
|
||||
|
||||
exportFromFullLedgerImpl(client.Client, startSeq, endSeq, path)
|
||||
}
|
||||
|
||||
func exportFromFullLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
|
||||
getLedgerFullData(client, startSeq, path)
|
||||
|
||||
//We need to fetch the ledger header and txs for startSeq as well
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from full ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
}
|
||||
|
||||
func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
}
|
||||
|
||||
defer client.Close()
|
||||
|
||||
exportFromDeltaLedgerImpl(client.Client, startSeq, endSeq, path)
|
||||
}
|
||||
|
||||
func exportFromDeltaLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
}
|
||||
153
tools/snapshot/internal/export/export_test.go
Normal file
153
tools/snapshot/internal/export/export_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"xrplf/clio/clio_snapshot/mocks"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
)
|
||||
|
||||
func TestExportDeltaLedgerData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
startSeq uint32
|
||||
endSeq uint32
|
||||
}{
|
||||
{"OneSeq", 1, 1},
|
||||
{"MultipleSeq", 1, 20},
|
||||
{"EndSeqLessThanStartSeq", 20, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
|
||||
mockClient := mocks.NewMockXRPLedgerAPIServiceClient(ctrl)
|
||||
|
||||
mockResponse := &pb.GetLedgerResponse{}
|
||||
|
||||
times := tt.endSeq - tt.startSeq + 1
|
||||
if tt.endSeq < tt.startSeq {
|
||||
times = 0
|
||||
}
|
||||
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(int(times))
|
||||
|
||||
defer os.RemoveAll("test")
|
||||
|
||||
exportFromDeltaLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
|
||||
|
||||
_, err := os.Stat("test")
|
||||
|
||||
assert.Equal(t, os.IsNotExist(err), tt.endSeq < tt.startSeq)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportFullLedgerData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
startSeq uint32
|
||||
endSeq uint32
|
||||
}{
|
||||
{"OneSeq", 1, 1},
|
||||
{"MultipleSeq", 1, 20},
|
||||
{"EndSeqLessThanStartSeq", 20, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
|
||||
mockClient := mocks.NewMockXRPLedgerAPIServiceClient(ctrl)
|
||||
|
||||
mockDataResponse := &pb.GetLedgerDataResponse{}
|
||||
mockLedgerResponse := &pb.GetLedgerResponse{}
|
||||
|
||||
timesLedgerDataCalled := 16
|
||||
timesLedgerCalled := tt.endSeq - tt.startSeq + 1
|
||||
|
||||
if tt.endSeq < tt.startSeq {
|
||||
timesLedgerCalled = 0
|
||||
}
|
||||
|
||||
mockClient.EXPECT().GetLedgerData(gomock.Any(), gomock.Any()).Return(mockDataResponse, nil).Times(timesLedgerDataCalled)
|
||||
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockLedgerResponse, nil).Times(int(timesLedgerCalled))
|
||||
|
||||
defer os.RemoveAll("test")
|
||||
exportFromFullLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
|
||||
|
||||
_, err := os.Stat("test")
|
||||
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundDown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in1 uint32
|
||||
in2 uint32
|
||||
out uint32
|
||||
}{
|
||||
{"RoundDownToZero", 10, 0, 10},
|
||||
{"RoundDown12To10", 12, 10, 10},
|
||||
{"RoundDownToOne", 13, 1, 13},
|
||||
{"RoundDown100", 103, 100, 100},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in uint32
|
||||
out [][32]byte
|
||||
}{
|
||||
{"GenerateMarkers1", 1, [][32]byte{{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}},
|
||||
{"GenerateMarkers2", 2, [][32]byte{{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
|
||||
{0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}},
|
||||
{"GenerateMarkers4", 4, [][32]byte{{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
|
||||
{0x40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
|
||||
{0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
|
||||
{0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, generateMarkers(tt.in), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"Path", "test"},
|
||||
{"NestedPath", "test/test"}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
checkPath(tt.path)
|
||||
defer os.RemoveAll(tt.path)
|
||||
_, err := os.Stat(tt.path)
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
63
tools/snapshot/internal/parse_args/parse_args.go
Normal file
63
tools/snapshot/internal/parse_args/parse_args.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package parse_args
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
flag "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
type Args struct {
|
||||
ExportMode string
|
||||
StartSeq uint32
|
||||
EndSeq uint32
|
||||
Path string
|
||||
GrpcServer string
|
||||
ServerMode bool
|
||||
GrpcPort uint32
|
||||
WsPort uint32
|
||||
}
|
||||
|
||||
func Parse() (*Args, error) {
|
||||
|
||||
fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
|
||||
fs.Usage = PrintUsage
|
||||
|
||||
exportMode := fs.StringP("export", "e", "", "Set export mode: 'full' (export full ledger data of start_seq) or 'delta' (only export ledger diff data)")
|
||||
seq := fs.Uint32("start_seq", 0, "Starting sequence number")
|
||||
endSeq := fs.Uint32("end_seq", 0, "Ending sequence number")
|
||||
path := fs.StringP("path", "p", "", "Path to the data")
|
||||
grpcServer := fs.StringP("grpc_server", "g", "localhost:50051", "rippled's gRPC server address")
|
||||
serverMode := fs.BoolP("server", "s", false, "Start server mode")
|
||||
grpcPort := fs.Uint32("grpc_port", 0, "Port for gRPC server to listen on")
|
||||
wsPort := fs.Uint32("ws_port", 0, "Port for WebSocket server to listen on")
|
||||
fs.Parse(os.Args[1:])
|
||||
|
||||
if *serverMode && *exportMode != "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --server and --export cannot be used at the same time.")
|
||||
}
|
||||
|
||||
if *serverMode {
|
||||
if *grpcPort == 0 || *wsPort == 0 || *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --grpc_port and --ws_port and --path are required for server mode.")
|
||||
}
|
||||
} else if *exportMode != "" {
|
||||
if *exportMode == "full" || *exportMode == "delta" {
|
||||
if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: --export or --server flag is required.")
|
||||
}
|
||||
|
||||
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *serverMode, *grpcPort, *wsPort}, nil
|
||||
}
|
||||
|
||||
func PrintUsage() {
|
||||
fmt.Println("Usage: clio_snapshot [options]")
|
||||
fmt.Println("Options:")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
94
tools/snapshot/internal/parse_args/parse_args_test.go
Normal file
94
tools/snapshot/internal/parse_args/parse_args_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package parse_args
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want *Args
|
||||
expectErr bool
|
||||
errMessage string
|
||||
}{
|
||||
{
|
||||
name: "Valid export full mode",
|
||||
args: []string{"cmd", "--export=full", "--start_seq=1", "--end_seq=10", "--path=/data", "--grpc_server=localhost:50051"},
|
||||
want: &Args{
|
||||
ExportMode: "full",
|
||||
StartSeq: 1,
|
||||
EndSeq: 10,
|
||||
Path: "/data",
|
||||
GrpcServer: "localhost:50051",
|
||||
ServerMode: false,
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
name: "Missing required flags in export mode",
|
||||
args: []string{"cmd", "--export=delta", "--start_seq=1"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
|
||||
},
|
||||
{
|
||||
name: "Invalid export mode",
|
||||
args: []string{"cmd", "--export=invalid"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: Invalid export mode. Use 'full' or 'delta'.",
|
||||
},
|
||||
{
|
||||
name: "Server mode with default grpc server flags",
|
||||
args: []string{"cmd", "--server", "--ws_port=1234", "--grpc_port=22", "--path=/server_data"},
|
||||
want: &Args{
|
||||
ServerMode: true,
|
||||
GrpcPort: 22,
|
||||
WsPort: 1234,
|
||||
StartSeq: 0,
|
||||
EndSeq: 0,
|
||||
Path: "/server_data",
|
||||
GrpcServer: "localhost:50051",
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
name: "Server and export mode together (error)",
|
||||
args: []string{"cmd", "--server", "--export=full"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
flag.CommandLine = flag.NewFlagSet(os.Args[1], flag.ExitOnError)
|
||||
|
||||
os.Args = tt.args
|
||||
|
||||
fmt.Println("Running test with args:", tt.args)
|
||||
got, err := Parse()
|
||||
|
||||
if tt.expectErr {
|
||||
if err == nil {
|
||||
t.Errorf("Expected error but got none")
|
||||
} else if err.Error() != tt.errMessage {
|
||||
t.Errorf("Expected error message '%s', got '%s'", tt.errMessage, err.Error())
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("Expected %+v, got %+v", tt.want, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
247
tools/snapshot/mocks/mock_grpc_server.go
Normal file
247
tools/snapshot/mocks/mock_grpc_server.go
Normal file
@@ -0,0 +1,247 @@
|
||||
// Code generated by MockGen. DO NOT EDIT.
|
||||
// Source: org/xrpl/rpc/v1/xrp_ledger_grpc.pb.go
|
||||
|
||||
// Package mocks is a generated GoMock package.
|
||||
package mocks
|
||||
|
||||
import (
|
||||
context "context"
|
||||
reflect "reflect"
|
||||
v1 "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
grpc "google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// MockXRPLedgerAPIServiceClient is a mock of XRPLedgerAPIServiceClient interface.
|
||||
type MockXRPLedgerAPIServiceClient struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockXRPLedgerAPIServiceClientMockRecorder
|
||||
}
|
||||
|
||||
// MockXRPLedgerAPIServiceClientMockRecorder is the mock recorder for MockXRPLedgerAPIServiceClient.
|
||||
type MockXRPLedgerAPIServiceClientMockRecorder struct {
|
||||
mock *MockXRPLedgerAPIServiceClient
|
||||
}
|
||||
|
||||
// NewMockXRPLedgerAPIServiceClient creates a new mock instance.
|
||||
func NewMockXRPLedgerAPIServiceClient(ctrl *gomock.Controller) *MockXRPLedgerAPIServiceClient {
|
||||
mock := &MockXRPLedgerAPIServiceClient{ctrl: ctrl}
|
||||
mock.recorder = &MockXRPLedgerAPIServiceClientMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use.
|
||||
func (m *MockXRPLedgerAPIServiceClient) EXPECT() *MockXRPLedgerAPIServiceClientMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// GetLedger mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceClient) GetLedger(ctx context.Context, in *v1.GetLedgerRequest, opts ...grpc.CallOption) (*v1.GetLedgerResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
varargs := []interface{}{ctx, in}
|
||||
for _, a := range opts {
|
||||
varargs = append(varargs, a)
|
||||
}
|
||||
ret := m.ctrl.Call(m, "GetLedger", varargs...)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedger indicates an expected call of GetLedger.
|
||||
func (mr *MockXRPLedgerAPIServiceClientMockRecorder) GetLedger(ctx, in interface{}, opts ...interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
varargs := append([]interface{}{ctx, in}, opts...)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedger", reflect.TypeOf((*MockXRPLedgerAPIServiceClient)(nil).GetLedger), varargs...)
|
||||
}
|
||||
|
||||
// GetLedgerData mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceClient) GetLedgerData(ctx context.Context, in *v1.GetLedgerDataRequest, opts ...grpc.CallOption) (*v1.GetLedgerDataResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
varargs := []interface{}{ctx, in}
|
||||
for _, a := range opts {
|
||||
varargs = append(varargs, a)
|
||||
}
|
||||
ret := m.ctrl.Call(m, "GetLedgerData", varargs...)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerDataResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerData indicates an expected call of GetLedgerData.
|
||||
func (mr *MockXRPLedgerAPIServiceClientMockRecorder) GetLedgerData(ctx, in interface{}, opts ...interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
varargs := append([]interface{}{ctx, in}, opts...)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerData", reflect.TypeOf((*MockXRPLedgerAPIServiceClient)(nil).GetLedgerData), varargs...)
|
||||
}
|
||||
|
||||
// GetLedgerDiff mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceClient) GetLedgerDiff(ctx context.Context, in *v1.GetLedgerDiffRequest, opts ...grpc.CallOption) (*v1.GetLedgerDiffResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
varargs := []interface{}{ctx, in}
|
||||
for _, a := range opts {
|
||||
varargs = append(varargs, a)
|
||||
}
|
||||
ret := m.ctrl.Call(m, "GetLedgerDiff", varargs...)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerDiffResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerDiff indicates an expected call of GetLedgerDiff.
|
||||
func (mr *MockXRPLedgerAPIServiceClientMockRecorder) GetLedgerDiff(ctx, in interface{}, opts ...interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
varargs := append([]interface{}{ctx, in}, opts...)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerDiff", reflect.TypeOf((*MockXRPLedgerAPIServiceClient)(nil).GetLedgerDiff), varargs...)
|
||||
}
|
||||
|
||||
// GetLedgerEntry mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceClient) GetLedgerEntry(ctx context.Context, in *v1.GetLedgerEntryRequest, opts ...grpc.CallOption) (*v1.GetLedgerEntryResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
varargs := []interface{}{ctx, in}
|
||||
for _, a := range opts {
|
||||
varargs = append(varargs, a)
|
||||
}
|
||||
ret := m.ctrl.Call(m, "GetLedgerEntry", varargs...)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerEntryResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerEntry indicates an expected call of GetLedgerEntry.
|
||||
func (mr *MockXRPLedgerAPIServiceClientMockRecorder) GetLedgerEntry(ctx, in interface{}, opts ...interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
varargs := append([]interface{}{ctx, in}, opts...)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerEntry", reflect.TypeOf((*MockXRPLedgerAPIServiceClient)(nil).GetLedgerEntry), varargs...)
|
||||
}
|
||||
|
||||
// MockXRPLedgerAPIServiceServer is a mock of XRPLedgerAPIServiceServer interface.
|
||||
type MockXRPLedgerAPIServiceServer struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockXRPLedgerAPIServiceServerMockRecorder
|
||||
}
|
||||
|
||||
// MockXRPLedgerAPIServiceServerMockRecorder is the mock recorder for MockXRPLedgerAPIServiceServer.
|
||||
type MockXRPLedgerAPIServiceServerMockRecorder struct {
|
||||
mock *MockXRPLedgerAPIServiceServer
|
||||
}
|
||||
|
||||
// NewMockXRPLedgerAPIServiceServer creates a new mock instance.
|
||||
func NewMockXRPLedgerAPIServiceServer(ctrl *gomock.Controller) *MockXRPLedgerAPIServiceServer {
|
||||
mock := &MockXRPLedgerAPIServiceServer{ctrl: ctrl}
|
||||
mock.recorder = &MockXRPLedgerAPIServiceServerMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use.
|
||||
func (m *MockXRPLedgerAPIServiceServer) EXPECT() *MockXRPLedgerAPIServiceServerMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// GetLedger mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceServer) GetLedger(arg0 context.Context, arg1 *v1.GetLedgerRequest) (*v1.GetLedgerResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetLedger", arg0, arg1)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedger indicates an expected call of GetLedger.
|
||||
func (mr *MockXRPLedgerAPIServiceServerMockRecorder) GetLedger(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedger", reflect.TypeOf((*MockXRPLedgerAPIServiceServer)(nil).GetLedger), arg0, arg1)
|
||||
}
|
||||
|
||||
// GetLedgerData mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceServer) GetLedgerData(arg0 context.Context, arg1 *v1.GetLedgerDataRequest) (*v1.GetLedgerDataResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetLedgerData", arg0, arg1)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerDataResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerData indicates an expected call of GetLedgerData.
|
||||
func (mr *MockXRPLedgerAPIServiceServerMockRecorder) GetLedgerData(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerData", reflect.TypeOf((*MockXRPLedgerAPIServiceServer)(nil).GetLedgerData), arg0, arg1)
|
||||
}
|
||||
|
||||
// GetLedgerDiff mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceServer) GetLedgerDiff(arg0 context.Context, arg1 *v1.GetLedgerDiffRequest) (*v1.GetLedgerDiffResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetLedgerDiff", arg0, arg1)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerDiffResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerDiff indicates an expected call of GetLedgerDiff.
|
||||
func (mr *MockXRPLedgerAPIServiceServerMockRecorder) GetLedgerDiff(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerDiff", reflect.TypeOf((*MockXRPLedgerAPIServiceServer)(nil).GetLedgerDiff), arg0, arg1)
|
||||
}
|
||||
|
||||
// GetLedgerEntry mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceServer) GetLedgerEntry(arg0 context.Context, arg1 *v1.GetLedgerEntryRequest) (*v1.GetLedgerEntryResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetLedgerEntry", arg0, arg1)
|
||||
ret0, _ := ret[0].(*v1.GetLedgerEntryResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetLedgerEntry indicates an expected call of GetLedgerEntry.
|
||||
func (mr *MockXRPLedgerAPIServiceServerMockRecorder) GetLedgerEntry(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLedgerEntry", reflect.TypeOf((*MockXRPLedgerAPIServiceServer)(nil).GetLedgerEntry), arg0, arg1)
|
||||
}
|
||||
|
||||
// mustEmbedUnimplementedXRPLedgerAPIServiceServer mocks base method.
|
||||
func (m *MockXRPLedgerAPIServiceServer) mustEmbedUnimplementedXRPLedgerAPIServiceServer() {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "mustEmbedUnimplementedXRPLedgerAPIServiceServer")
|
||||
}
|
||||
|
||||
// mustEmbedUnimplementedXRPLedgerAPIServiceServer indicates an expected call of mustEmbedUnimplementedXRPLedgerAPIServiceServer.
|
||||
func (mr *MockXRPLedgerAPIServiceServerMockRecorder) mustEmbedUnimplementedXRPLedgerAPIServiceServer() *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedXRPLedgerAPIServiceServer", reflect.TypeOf((*MockXRPLedgerAPIServiceServer)(nil).mustEmbedUnimplementedXRPLedgerAPIServiceServer))
|
||||
}
|
||||
|
||||
// MockUnsafeXRPLedgerAPIServiceServer is a mock of UnsafeXRPLedgerAPIServiceServer interface.
|
||||
type MockUnsafeXRPLedgerAPIServiceServer struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *MockUnsafeXRPLedgerAPIServiceServerMockRecorder
|
||||
}
|
||||
|
||||
// MockUnsafeXRPLedgerAPIServiceServerMockRecorder is the mock recorder for MockUnsafeXRPLedgerAPIServiceServer.
|
||||
type MockUnsafeXRPLedgerAPIServiceServerMockRecorder struct {
|
||||
mock *MockUnsafeXRPLedgerAPIServiceServer
|
||||
}
|
||||
|
||||
// NewMockUnsafeXRPLedgerAPIServiceServer creates a new mock instance.
|
||||
func NewMockUnsafeXRPLedgerAPIServiceServer(ctrl *gomock.Controller) *MockUnsafeXRPLedgerAPIServiceServer {
|
||||
mock := &MockUnsafeXRPLedgerAPIServiceServer{ctrl: ctrl}
|
||||
mock.recorder = &MockUnsafeXRPLedgerAPIServiceServerMockRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
// EXPECT returns an object that allows the caller to indicate expected use.
|
||||
func (m *MockUnsafeXRPLedgerAPIServiceServer) EXPECT() *MockUnsafeXRPLedgerAPIServiceServerMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// mustEmbedUnimplementedXRPLedgerAPIServiceServer mocks base method.
|
||||
func (m *MockUnsafeXRPLedgerAPIServiceServer) mustEmbedUnimplementedXRPLedgerAPIServiceServer() {
|
||||
m.ctrl.T.Helper()
|
||||
m.ctrl.Call(m, "mustEmbedUnimplementedXRPLedgerAPIServiceServer")
|
||||
}
|
||||
|
||||
// mustEmbedUnimplementedXRPLedgerAPIServiceServer indicates an expected call of mustEmbedUnimplementedXRPLedgerAPIServiceServer.
|
||||
func (mr *MockUnsafeXRPLedgerAPIServiceServerMockRecorder) mustEmbedUnimplementedXRPLedgerAPIServiceServer() *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedXRPLedgerAPIServiceServer", reflect.TypeOf((*MockUnsafeXRPLedgerAPIServiceServer)(nil).mustEmbedUnimplementedXRPLedgerAPIServiceServer))
|
||||
}
|
||||
23
tools/snapshot/snapshot.go
Normal file
23
tools/snapshot/snapshot.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/export"
|
||||
"xrplf/clio/clio_snapshot/internal/parse_args"
|
||||
)
|
||||
|
||||
func main() {
|
||||
args, err := parse_args.Parse()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if args.ExportMode == "full" {
|
||||
export.ExportFromFullLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
|
||||
|
||||
} else if args.ExportMode == "delta" {
|
||||
export.ExportFromDeltaLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
|
||||
}
|
||||
//TODO: Implement server mode
|
||||
}
|
||||
Reference in New Issue
Block a user