mirror of
https://github.com/XRPLF/clio.git
synced 2025-11-19 11:15:50 +00:00
feat: Snapshot import feature (#1970)
Implement snapshot import cmd `clio_snapshot --server --grpc_server 0.0.0.0:12345 --path <snapshot_path>` Implement snapshot range cmd `./clio_snapshot --range --path <snapshot_path>` Add LedgerHouses: It is responsible for reading/writing snapshot data Server: Start grpc server and ws server
This commit is contained in:
@@ -6,6 +6,7 @@ set(GO_SOURCE_DIR "${CMAKE_SOURCE_DIR}/tools/snapshot")
|
||||
set(PROTO_INC_DIR "${xrpl_PACKAGE_FOLDER_RELEASE}/include/xrpl/proto")
|
||||
set(PROTO_SOURCE_DIR "${PROTO_INC_DIR}/org/xrpl/rpc/v1/")
|
||||
set(GO_OUTPUT "${CMAKE_BINARY_DIR}/clio_snapshot")
|
||||
file(GLOB_RECURSE GO_SOURCES ${GO_SOURCE_DIR}/*.go)
|
||||
|
||||
set(PROTO_FILES
|
||||
${PROTO_SOURCE_DIR}/xrp_ledger.proto ${PROTO_SOURCE_DIR}/ledger.proto ${PROTO_SOURCE_DIR}/get_ledger.proto
|
||||
@@ -34,7 +35,7 @@ endforeach()
|
||||
foreach (proto ${PROTO_FILES})
|
||||
get_filename_component(proto_name ${proto} NAME_WE)
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_SOURCE_DIR}/${proto_name}.pb.go
|
||||
OUTPUT ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go
|
||||
COMMAND
|
||||
protoc ${GO_OPTS} ${GRPC_OPTS}
|
||||
--go-grpc_out=${GO_SOURCE_DIR} -I${PROTO_INC_DIR} ${proto} --plugin=${GOPATH_VALUE}/bin/protoc-gen-go
|
||||
@@ -44,7 +45,7 @@ foreach (proto ${PROTO_FILES})
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${proto_name}.pb.go)
|
||||
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go)
|
||||
endforeach ()
|
||||
|
||||
add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
|
||||
@@ -52,15 +53,16 @@ add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
|
||||
add_custom_target(run_go_tests
|
||||
COMMAND go test ./...
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
DEPENDS ${GENERATED_GO_FILES}
|
||||
COMMENT "Running clio_snapshot unittests"
|
||||
VERBATIM
|
||||
DEPENDS ${GENERATED_GO_FILES}
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_OUTPUT}
|
||||
COMMAND ${GO_EXECUTABLE} build -o ${GO_OUTPUT} ${GO_SOURCE_DIR}
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
DEPENDS ${GO_SOURCES}
|
||||
COMMENT "Building clio_snapshot"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ toolchain go1.22.11
|
||||
|
||||
require (
|
||||
github.com/golang/mock v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/spf13/pflag v1.0.6
|
||||
github.com/stretchr/testify v1.10.0
|
||||
google.golang.org/grpc v1.69.4
|
||||
|
||||
@@ -12,6 +12,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
|
||||
@@ -4,20 +4,20 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
deltaDataFolderDiv = 10000
|
||||
grpcUser = "clio-snapshot"
|
||||
markerNum = 16
|
||||
grpcUser = "clio-snapshot"
|
||||
markerNum = 16
|
||||
maxConcurrency = 256
|
||||
firstAvailableLedger = 32570
|
||||
)
|
||||
|
||||
type gRPCClient struct {
|
||||
@@ -44,7 +44,25 @@ func createGRPCClient(serverAddr string) (*gRPCClient, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
func getLedgerDeltaDataInParallel(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, ledgersHouse *ledgers.LedgersHouse) {
|
||||
sem := make(chan struct{}, maxConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
|
||||
go func(seq uint32) {
|
||||
defer wg.Done()
|
||||
log.Printf("Process delta sequence: %d\n", seq)
|
||||
getLedgerDeltaData(client, seq, ledgersHouse)
|
||||
<-sem
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgersHouse *ledgers.LedgersHouse) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -56,49 +74,27 @@ func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path st
|
||||
}
|
||||
request.Ledger = ledger
|
||||
request.User = grpcUser
|
||||
request.GetObjectNeighbors = true
|
||||
request.Transactions = true
|
||||
request.Expand = true
|
||||
request.GetObjects = true
|
||||
|
||||
// The first available ledger doesn't have diff data
|
||||
request.GetObjects = firstAvailableLedger != seq
|
||||
request.GetObjectNeighbors = firstAvailableLedger != seq
|
||||
|
||||
response, err := client.GetLedger(ctx, &request)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
log.Fatalf("Error getting ledger delta data: %v - seq: %d", err, seq)
|
||||
}
|
||||
|
||||
saveLedgerDeltaData(seq, response, path)
|
||||
err = ledgersHouse.WriteLedgerDeltaData(seq, response)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing ledger delta data: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Processing delta sequence: %d\n", seq)
|
||||
}
|
||||
|
||||
func roundDown(n uint32, roundTo uint32) uint32 {
|
||||
if roundTo == 0 {
|
||||
return n
|
||||
}
|
||||
return n - (n % roundTo)
|
||||
}
|
||||
|
||||
func saveLedgerDeltaData(seq uint32, response *pb.GetLedgerResponse, path string) {
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
protoData, err := proto.Marshal(response)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
|
||||
|
||||
err = os.WriteFile(filePath, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func generateMarkers(markerNum uint32) [][32]byte {
|
||||
var byteArray [32]byte
|
||||
|
||||
@@ -114,19 +110,7 @@ func generateMarkers(markerNum uint32) [][32]byte {
|
||||
return byteArrayList
|
||||
}
|
||||
|
||||
func saveLedgerData(path string, data *pb.GetLedgerDataResponse) {
|
||||
protoData, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, path string) {
|
||||
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, ledgerHouse *ledgers.LedgersHouse) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -143,25 +127,22 @@ func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byt
|
||||
}
|
||||
request.User = grpcUser
|
||||
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%x", marker))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
for request.Marker != nil {
|
||||
res, err := client.GetLedgerData(ctx, &request)
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%x.dat", request.Marker))
|
||||
saveLedgerData(filePath, res)
|
||||
err = ledgerHouse.WriteLedgerData(seq, request.Marker, res)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing ledger data: %v", err)
|
||||
}
|
||||
log.Printf("Saving ledger data %x", request.Marker)
|
||||
request.Marker = res.Marker
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHouse *ledgers.LedgersHouse) {
|
||||
log.Printf("Processing full sequence: %d\n", seq)
|
||||
|
||||
markers := generateMarkers(markerNum)
|
||||
@@ -176,11 +157,9 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
|
||||
end = markers[i+1][:]
|
||||
}
|
||||
|
||||
fmt.Printf("Got ledger data marker: %x-%x\n", marker, end)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
getLedgerData(client, seq, marker[:], end, path)
|
||||
getLedgerData(client, seq, marker[:], end, ledgerHouse)
|
||||
}()
|
||||
|
||||
}
|
||||
@@ -188,19 +167,7 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func checkPath(path string) {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
// Create the directory if it doesn't exist
|
||||
err := os.MkdirAll(path, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
@@ -212,20 +179,21 @@ func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, pat
|
||||
}
|
||||
|
||||
func exportFromFullLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
getLedgerFullData(client, startSeq, path)
|
||||
getLedgerFullData(client, startSeq, ledgersHouse)
|
||||
|
||||
//We need to fetch the ledger header and txs for startSeq as well
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
|
||||
|
||||
err := ledgersHouse.SetRange(startSeq, endSeq)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing range: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from full ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
}
|
||||
|
||||
func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
@@ -237,8 +205,27 @@ func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, pa
|
||||
}
|
||||
|
||||
func exportFromDeltaLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
_, oldEnd, err := ledgersHouse.GetRange()
|
||||
if err != nil {
|
||||
log.Fatalf("Can't find existing snapshot to extend: %v", err)
|
||||
}
|
||||
|
||||
if oldEnd < startSeq-1 {
|
||||
log.Fatalf("Missing delta ledger from %d to %d", oldEnd, startSeq)
|
||||
}
|
||||
|
||||
if oldEnd >= endSeq {
|
||||
log.Fatalf("The snapshot already contains the requested delta ledger")
|
||||
}
|
||||
|
||||
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
|
||||
|
||||
err = ledgersHouse.AppendDeltaLedger(startSeq, endSeq)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing new range: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
|
||||
@@ -1,26 +1,48 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
"xrplf/clio/clio_snapshot/mocks"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
)
|
||||
|
||||
// Matcher used to verify the GetLedgerRequest parameters
|
||||
type LedgerRequestMatcher struct {
|
||||
expectedObjects bool
|
||||
expectedNeighbors bool
|
||||
}
|
||||
|
||||
func (m LedgerRequestMatcher) Matches(x interface{}) bool {
|
||||
req, ok := x.(*pb.GetLedgerRequest)
|
||||
return ok && req.GetObjects == m.expectedObjects && req.GetObjectNeighbors == m.expectedNeighbors
|
||||
}
|
||||
|
||||
func (m LedgerRequestMatcher) String() string {
|
||||
return fmt.Sprintf("LedgerRequest with objects=%v neighbors=%v", m.expectedObjects, m.expectedNeighbors)
|
||||
}
|
||||
|
||||
func matchObjectsEquals(objects bool, neighbors bool) gomock.Matcher {
|
||||
return LedgerRequestMatcher{objects, neighbors}
|
||||
}
|
||||
|
||||
func TestExportDeltaLedgerData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
startSeq uint32
|
||||
endSeq uint32
|
||||
}{
|
||||
{"OneSeq", 1, 1},
|
||||
{"MultipleSeq", 1, 20},
|
||||
{"EndSeqLessThanStartSeq", 20, 1},
|
||||
{"OneSeq", 700000, 700000},
|
||||
{"MultipleSeq", 700000, 700019},
|
||||
{"FirstAvailableLedger", firstAvailableLedger, firstAvailableLedger},
|
||||
{"FirstAvailableLedgerMultipleSeq", firstAvailableLedger, firstAvailableLedger + 2},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -37,15 +59,26 @@ func TestExportDeltaLedgerData(t *testing.T) {
|
||||
times = 0
|
||||
}
|
||||
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(int(times))
|
||||
if tt.startSeq == firstAvailableLedger {
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(false, false)).Return(mockResponse, nil).Times(1)
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times) - 1)
|
||||
} else {
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times))
|
||||
}
|
||||
|
||||
os.MkdirAll("test", os.ModePerm)
|
||||
|
||||
manifest := ledgers.NewManifest("test")
|
||||
manifest.SetLedgerRange(tt.startSeq-1, tt.startSeq-1)
|
||||
|
||||
defer os.RemoveAll("test")
|
||||
|
||||
exportFromDeltaLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
|
||||
|
||||
_, err := os.Stat("test")
|
||||
|
||||
assert.Equal(t, os.IsNotExist(err), tt.endSeq < tt.startSeq)
|
||||
seq1, seq2, err := manifest.Read()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, tt.startSeq-1, seq1)
|
||||
assert.Equal(t, tt.endSeq, seq2)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -92,26 +125,6 @@ func TestExportFullLedgerData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundDown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in1 uint32
|
||||
in2 uint32
|
||||
out uint32
|
||||
}{
|
||||
{"RoundDownToZero", 10, 0, 10},
|
||||
{"RoundDown12To10", 12, 10, 10},
|
||||
{"RoundDownToOne", 13, 1, 13},
|
||||
{"RoundDown100", 103, 100, 100},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -132,22 +145,3 @@ func TestGenerateMarkers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"Path", "test"},
|
||||
{"NestedPath", "test/test"}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
checkPath(tt.path)
|
||||
defer os.RemoveAll(tt.path)
|
||||
_, err := os.Stat(tt.path)
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
133
tools/snapshot/internal/ledgers/ledgers.go
Normal file
133
tools/snapshot/internal/ledgers/ledgers.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
const deltaDataFolderDiv = 10000
|
||||
const readWritePerm = 0644
|
||||
|
||||
func convertInnerMarkerToMarker(in []byte) []byte {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(in))
|
||||
out[0] = in[0] & 0xf0
|
||||
return out
|
||||
}
|
||||
|
||||
func checkPath(path string) error {
|
||||
dir := filepath.Dir(path)
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
// Create the directory if it doesn't exist
|
||||
err := os.MkdirAll(dir, os.ModePerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error creating directory: %s,%v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func roundDown(n uint32, roundTo uint32) uint32 {
|
||||
if roundTo == 0 {
|
||||
return n
|
||||
}
|
||||
return n - (n % roundTo)
|
||||
}
|
||||
|
||||
type LedgersHouse struct {
|
||||
path string
|
||||
manifest *Manifest
|
||||
}
|
||||
|
||||
func NewLedgersHouse(path string) *LedgersHouse {
|
||||
return &LedgersHouse{path: path, manifest: NewManifest(path)}
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) deltaDataPath(seq uint32) string {
|
||||
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
|
||||
return filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) fullDataPath(seq uint32, marker string, innerMarker string) string {
|
||||
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%s", marker))
|
||||
return filepath.Join(subPath, fmt.Sprintf("%s.dat", innerMarker))
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) ReadLedgerDeltaData(seq uint32) (*pb.GetLedgerResponse, error) {
|
||||
path := lh.deltaDataPath(seq)
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ledger pb.GetLedgerResponse
|
||||
if err := proto.Unmarshal(data, &ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ledger, nil
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) WriteLedgerDeltaData(seq uint32, data *pb.GetLedgerResponse) error {
|
||||
path := lh.deltaDataPath(seq)
|
||||
err := checkPath(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataBytes, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, dataBytes, readWritePerm)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) ReadLedgerData(seq uint32, innerMarker []byte) (*pb.GetLedgerDataResponse, error) {
|
||||
marker := convertInnerMarkerToMarker(innerMarker)
|
||||
path := lh.fullDataPath(seq, fmt.Sprintf("%x", marker), fmt.Sprintf("%x", innerMarker))
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ledger pb.GetLedgerDataResponse
|
||||
if err := proto.Unmarshal(data, &ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ledger, nil
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) WriteLedgerData(seq uint32, innerMarker []byte, data *pb.GetLedgerDataResponse) error {
|
||||
path := lh.fullDataPath(seq, fmt.Sprintf("%x", convertInnerMarkerToMarker(innerMarker)), fmt.Sprintf("%x", innerMarker))
|
||||
err := checkPath(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataBytes, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, dataBytes, readWritePerm)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) SetRange(startSeq uint32, endSeq uint32) error {
|
||||
return lh.manifest.SetLedgerRange(startSeq, endSeq)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) AppendDeltaLedger(startSeq uint32, endSeq uint32) error {
|
||||
return lh.manifest.AppendDeltaLedger(startSeq, endSeq)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) IsExist() bool {
|
||||
return lh.manifest.IsExist()
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) GetRange() (uint32, uint32, error) {
|
||||
return lh.manifest.Read()
|
||||
}
|
||||
188
tools/snapshot/internal/ledgers/ledgers_test.go
Normal file
188
tools/snapshot/internal/ledgers/ledgers_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"Path", "test/d.dat"},
|
||||
{"NestedPath", "test/test/d.dat"}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := checkPath(tt.path)
|
||||
assert.NoError(t, err)
|
||||
dir := filepath.Dir(tt.path)
|
||||
defer os.RemoveAll("test")
|
||||
_, err = os.Stat(dir)
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestRoundDown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in1 uint32
|
||||
in2 uint32
|
||||
out uint32
|
||||
}{
|
||||
{"RoundDownToZero", 10, 0, 10},
|
||||
{"RoundDown12To10", 12, 10, 10},
|
||||
{"RoundDownToOne", 13, 1, 13},
|
||||
{"RoundDown100", 103, 100, 100},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertInnerMarkerToMarker(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
in []byte
|
||||
out []byte
|
||||
}{
|
||||
{"SingleByte", []byte{0x01}, []byte{0x00}},
|
||||
{"MultipleBytes", []byte{0x01, 0x02, 0x03}, []byte{0x00, 0x00, 0x00}},
|
||||
{"MultipleBytes2", []byte{0xf1, 0x02, 0x03}, []byte{0xf0, 0x00, 0x00}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, convertInnerMarkerToMarker(tt.in), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLedgersHouseGetDeltaPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("testdata")
|
||||
assert.Equal(t, lh.deltaDataPath(12345), "testdata/ledger_diff_10000/12345.dat")
|
||||
|
||||
assert.Equal(t, lh.deltaDataPath(3), "testdata/ledger_diff_0/3.dat")
|
||||
|
||||
assert.Equal(t, lh.deltaDataPath(0), "testdata/ledger_diff_0/0.dat")
|
||||
}
|
||||
|
||||
func TestLedgersHouseGetFullDataPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("testdata")
|
||||
assert.Equal(t, lh.fullDataPath(12345, "fffff", "ababab"), "testdata/ledger_data_12345/marker_fffff/ababab.dat")
|
||||
}
|
||||
|
||||
func TestLedgerHouseLedgerDeltaData(t *testing.T) {
|
||||
defer os.RemoveAll("testdata")
|
||||
lh := NewLedgersHouse("testdata")
|
||||
data, err := lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
|
||||
data, err = lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data != nil)
|
||||
assert.True(t, err == nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseInvalidLedgerDeltaPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("/etc")
|
||||
data, err := lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
err = lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
|
||||
assert.True(t, err != nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseLedgerData(t *testing.T) {
|
||||
defer os.RemoveAll("testdata")
|
||||
lh := NewLedgersHouse("testdata")
|
||||
data, err := lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
|
||||
data, err = lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data != nil)
|
||||
assert.True(t, err == nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseInvalidLedgerDataPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("/etc")
|
||||
data, err := lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
err = lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
|
||||
assert.True(t, err != nil)
|
||||
}
|
||||
|
||||
func TestLedgersHouseManifest(t *testing.T) {
|
||||
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
lh := NewLedgersHouse("testdata")
|
||||
startSeq, endSeq, err := lh.GetRange()
|
||||
assert.True(t, err != nil)
|
||||
assert.Equal(t, startSeq, uint32(0))
|
||||
assert.Equal(t, endSeq, uint32(0))
|
||||
assert.False(t, lh.IsExist())
|
||||
|
||||
lh.SetRange(1, 100)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(100))
|
||||
|
||||
lh.AppendDeltaLedger(100, 200)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(200))
|
||||
|
||||
lh.AppendDeltaLedger(201, 300)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(201, 100)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(302, 350)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(0, 350)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
}
|
||||
91
tools/snapshot/internal/ledgers/manifest.go
Normal file
91
tools/snapshot/internal/ledgers/manifest.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
fileName = "manifest.txt"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
folderPath string
|
||||
filePath string
|
||||
}
|
||||
|
||||
func NewManifest(folderPath string) *Manifest {
|
||||
return &Manifest{
|
||||
folderPath: folderPath,
|
||||
filePath: filepath.Join(folderPath, fileName),
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *Manifest) SetLedgerRange(start uint32, end uint32) error {
|
||||
content := fmt.Sprintf("%d|%d", start, end)
|
||||
return fm.writeToFile(content)
|
||||
}
|
||||
|
||||
func (fm *Manifest) AppendDeltaLedger(delta1 uint32, delta2 uint32) error {
|
||||
start, end, err := fm.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
//rewrite the range if new delta can extend the current range continuously
|
||||
if delta1 >= start && (end+1) >= delta1 && delta2 >= delta1 {
|
||||
return fm.SetLedgerRange(start, delta2)
|
||||
}
|
||||
|
||||
return fmt.Errorf("Invalid delta ledger range")
|
||||
}
|
||||
|
||||
func (fm *Manifest) writeToFile(content string) error {
|
||||
os.MkdirAll(fm.folderPath, os.ModePerm)
|
||||
file, err := os.OpenFile(fm.filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.WriteString(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fm *Manifest) IsExist() bool {
|
||||
_, err := os.Stat(fm.filePath)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
func (fm *Manifest) Read() (uint32, uint32, error) {
|
||||
content, err := os.ReadFile(fm.filePath)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(content) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
parts := strings.Split(string(content), "|")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("file content is not in expected format")
|
||||
}
|
||||
|
||||
part1, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 32)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error parsing the first part: %v", err)
|
||||
}
|
||||
|
||||
part2, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 32)
|
||||
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error parsing the second part: %v", err)
|
||||
}
|
||||
|
||||
return uint32(part1), uint32(part2), nil
|
||||
}
|
||||
41
tools/snapshot/internal/ledgers/manifest_test.go
Normal file
41
tools/snapshot/internal/ledgers/manifest_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestManifest(t *testing.T) {
|
||||
manifest := NewManifest("testdata")
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
assert.False(t, manifest.IsExist())
|
||||
_, _, err := manifest.Read()
|
||||
assert.Error(t, err)
|
||||
|
||||
err = manifest.SetLedgerRange(1, 10)
|
||||
assert.NoError(t, err)
|
||||
err = manifest.AppendDeltaLedger(11, 20)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, manifest.IsExist())
|
||||
err = manifest.AppendDeltaLedger(22, 30)
|
||||
assert.Error(t, err)
|
||||
|
||||
start, end, err := manifest.Read()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, start, uint32(1))
|
||||
assert.Equal(t, end, uint32(20))
|
||||
}
|
||||
|
||||
func TestManifestInvalidPath(t *testing.T) {
|
||||
manifest := NewManifest("/")
|
||||
|
||||
assert.False(t, manifest.IsExist())
|
||||
_, _, err := manifest.Read()
|
||||
assert.Error(t, err)
|
||||
|
||||
err = manifest.SetLedgerRange(1, 10)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -13,9 +13,9 @@ type Args struct {
|
||||
EndSeq uint32
|
||||
Path string
|
||||
GrpcServer string
|
||||
WsServer string
|
||||
ServerMode bool
|
||||
GrpcPort uint32
|
||||
WsPort uint32
|
||||
ShowRange bool
|
||||
}
|
||||
|
||||
func Parse() (*Args, error) {
|
||||
@@ -27,10 +27,10 @@ func Parse() (*Args, error) {
|
||||
seq := fs.Uint32("start_seq", 0, "Starting sequence number")
|
||||
endSeq := fs.Uint32("end_seq", 0, "Ending sequence number")
|
||||
path := fs.StringP("path", "p", "", "Path to the data")
|
||||
grpcServer := fs.StringP("grpc_server", "g", "localhost:50051", "rippled's gRPC server address")
|
||||
grpcServer := fs.StringP("grpc_server", "g", "0.0.0.0:50051", "rippled's gRPC server address")
|
||||
wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address")
|
||||
serverMode := fs.BoolP("server", "s", false, "Start server mode")
|
||||
grpcPort := fs.Uint32("grpc_port", 0, "Port for gRPC server to listen on")
|
||||
wsPort := fs.Uint32("ws_port", 0, "Port for WebSocket server to listen on")
|
||||
showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot")
|
||||
fs.Parse(os.Args[1:])
|
||||
|
||||
if *serverMode && *exportMode != "" {
|
||||
@@ -38,22 +38,26 @@ func Parse() (*Args, error) {
|
||||
}
|
||||
|
||||
if *serverMode {
|
||||
if *grpcPort == 0 || *wsPort == 0 || *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --grpc_port and --ws_port and --path are required for server mode.")
|
||||
if *grpcServer == "" || *wsServer == "" || *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.")
|
||||
}
|
||||
} else if *exportMode != "" {
|
||||
if *exportMode == "full" || *exportMode == "delta" {
|
||||
if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
|
||||
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
|
||||
}
|
||||
} else if *showRange {
|
||||
if *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --path is required for show range.")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: --export or --server flag is required.")
|
||||
return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.")
|
||||
}
|
||||
|
||||
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *serverMode, *grpcPort, *wsPort}, nil
|
||||
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil
|
||||
}
|
||||
|
||||
func PrintUsage() {
|
||||
|
||||
@@ -25,6 +25,7 @@ func TestParse(t *testing.T) {
|
||||
EndSeq: 10,
|
||||
Path: "/data",
|
||||
GrpcServer: "localhost:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
ServerMode: false,
|
||||
},
|
||||
expectErr: false,
|
||||
@@ -34,7 +35,7 @@ func TestParse(t *testing.T) {
|
||||
args: []string{"cmd", "--export=delta", "--start_seq=1"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
|
||||
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.",
|
||||
},
|
||||
{
|
||||
name: "Invalid export mode",
|
||||
@@ -45,18 +46,24 @@ func TestParse(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "Server mode with default grpc server flags",
|
||||
args: []string{"cmd", "--server", "--ws_port=1234", "--grpc_port=22", "--path=/server_data"},
|
||||
args: []string{"cmd", "--server", "--path=/server_data"},
|
||||
want: &Args{
|
||||
ServerMode: true,
|
||||
GrpcPort: 22,
|
||||
WsPort: 1234,
|
||||
StartSeq: 0,
|
||||
EndSeq: 0,
|
||||
Path: "/server_data",
|
||||
GrpcServer: "localhost:50051",
|
||||
GrpcServer: "0.0.0.0:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
name: "Server mode with empty grpc server flag",
|
||||
args: []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.",
|
||||
},
|
||||
{
|
||||
name: "Server and export mode together (error)",
|
||||
args: []string{"cmd", "--server", "--export=full"},
|
||||
@@ -64,6 +71,24 @@ func TestParse(t *testing.T) {
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
|
||||
},
|
||||
{
|
||||
name: "Show range without path",
|
||||
args: []string{"cmd", "--range"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --path is required for show range.",
|
||||
},
|
||||
{
|
||||
name: "Show range",
|
||||
args: []string{"cmd", "--range", "--path=/range_data"},
|
||||
want: &Args{
|
||||
ShowRange: true,
|
||||
Path: "/range_data",
|
||||
GrpcServer: "0.0.0.0:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
41
tools/snapshot/internal/server/grpc_server.go
Normal file
41
tools/snapshot/internal/server/grpc_server.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
// create a server implement the xrpl rpc v1 server interface
|
||||
type Server struct {
|
||||
pb.XRPLedgerAPIServiceServer
|
||||
ledgersHouse *ledgers.LedgersHouse
|
||||
}
|
||||
|
||||
func (s *Server) GetLedger(ctx context.Context, req *pb.GetLedgerRequest) (*pb.GetLedgerResponse, error) {
|
||||
return s.ledgersHouse.ReadLedgerDeltaData(req.GetLedger().GetSequence())
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerData(ctx context.Context, req *pb.GetLedgerDataRequest) (*pb.GetLedgerDataResponse, error) {
|
||||
marker := req.GetMarker()
|
||||
if marker == nil {
|
||||
marker = make([]byte, 32)
|
||||
}
|
||||
return s.ledgersHouse.ReadLedgerData(req.GetLedger().GetSequence(), marker)
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerDiff(ctx context.Context, req *pb.GetLedgerDiffRequest) (*pb.GetLedgerDiffResponse, error) {
|
||||
return nil, fmt.Errorf("GetLedgerDiff not supported")
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerEntry(ctx context.Context, req *pb.GetLedgerEntryRequest) (*pb.GetLedgerEntryResponse, error) {
|
||||
return nil, fmt.Errorf("GetLedgerEntry not supported")
|
||||
}
|
||||
|
||||
func newServer(path string) *Server {
|
||||
s := &Server{}
|
||||
s.ledgersHouse = ledgers.NewLedgersHouse(path)
|
||||
return s
|
||||
}
|
||||
88
tools/snapshot/internal/server/grpc_server_test.go
Normal file
88
tools/snapshot/internal/server/grpc_server_test.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
func TestUnavaibleMethods(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
|
||||
req := &pb.GetLedgerDiffRequest{}
|
||||
_, err := srv.GetLedgerDiff(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "GetLedgerDiff not supported")
|
||||
|
||||
req2 := &pb.GetLedgerEntryRequest{}
|
||||
_, err = srv.GetLedgerEntry(context.Background(), req2)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "GetLedgerEntry not supported")
|
||||
}
|
||||
|
||||
func TestWhenPathIsInvalid(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
|
||||
req := &pb.GetLedgerRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := srv.GetLedger(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "open testdata/ledger_diff_0/2.dat: no such file or directory")
|
||||
|
||||
req2 := &pb.GetLedgerDataRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = srv.GetLedgerData(context.Background(), req2)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "open testdata/ledger_data_2/marker_0000000000000000000000000000000000000000000000000000000000000000/0000000000000000000000000000000000000000000000000000000000000000.dat: no such file or directory")
|
||||
}
|
||||
|
||||
func TestWhenPathIsValid(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
ledger := ledgers.NewLedgersHouse("testdata")
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
marker := [32]byte{}
|
||||
ledger.WriteLedgerData(1, marker[:], &pb.GetLedgerDataResponse{})
|
||||
ledger.WriteLedgerDeltaData(1, &pb.GetLedgerResponse{})
|
||||
|
||||
req := &pb.GetLedgerRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
res, err := srv.GetLedger(context.Background(), req)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res)
|
||||
|
||||
req2 := &pb.GetLedgerDataRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
res2, err := srv.GetLedgerData(context.Background(), req2)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res2)
|
||||
}
|
||||
46
tools/snapshot/internal/server/server.go
Normal file
46
tools/snapshot/internal/server/server.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func StartServer(grpcServerAddr string, wsServerAddr string, path string) {
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
if !ledgersHouse.IsExist() {
|
||||
log.Fatalf("Can't start server againist invalid snapshot folder: %s", path)
|
||||
}
|
||||
|
||||
startSeq, endSeq, err := ledgersHouse.GetRange()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get range: %v", err)
|
||||
}
|
||||
|
||||
lis, err := net.Listen("tcp", grpcServerAddr)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to listen: %v", err)
|
||||
}
|
||||
|
||||
grpcServer := grpc.NewServer()
|
||||
pb.RegisterXRPLedgerAPIServiceServer(grpcServer, newServer(path))
|
||||
log.Print("Starting server...")
|
||||
go grpcServer.Serve(lis)
|
||||
|
||||
wsServer := NewWebSocketServer("Snapshot Server", func(message string) string {
|
||||
//mimic the response of the ledger stream
|
||||
ledgerStreamReply := fmt.Sprintf("{\"fee_base\":10,\"ledger_hash\":\"A320C67DA7D1250A577AC5AACDF06ADC25E0EEEF7AE5B8D63CE2E1CC7F76A438\",\"ledger_index\":%d,\"ledger_time\":792853443,\"reserve_base\":1000000,\"reserve_inc\":200000,\"txn_count\":0,\"type\":\"ledgerClosed\",\"validated_ledgers\":\"%d-%d\"}",
|
||||
endSeq, startSeq, endSeq)
|
||||
return ledgerStreamReply
|
||||
})
|
||||
wsServer.Start(wsServerAddr)
|
||||
|
||||
select {}
|
||||
}
|
||||
64
tools/snapshot/internal/server/ws_server.go
Normal file
64
tools/snapshot/internal/server/ws_server.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type WebSocketServer struct {
|
||||
serverName string
|
||||
callback func(message string) string
|
||||
upgrader websocket.Upgrader
|
||||
}
|
||||
|
||||
func NewWebSocketServer(serverName string, callback func(message string) string) *WebSocketServer {
|
||||
return &WebSocketServer{
|
||||
serverName: serverName,
|
||||
callback: callback,
|
||||
upgrader: websocket.Upgrader{
|
||||
CheckOrigin: func(r *http.Request) bool { return true }, // Allow all connections
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (ws *WebSocketServer) handleConnections() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := ws.upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error upgrading to WebSocket: %v", ws.serverName, err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
log.Printf("[%s] New WebSocket connection established", ws.serverName)
|
||||
|
||||
for {
|
||||
_, msg, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error reading message: %v", ws.serverName, err)
|
||||
break
|
||||
}
|
||||
log.Printf("[%s] Received: %s", ws.serverName, msg)
|
||||
|
||||
response := ws.callback(string(msg))
|
||||
|
||||
err = conn.WriteMessage(websocket.TextMessage, []byte(response))
|
||||
log.Printf("[%s] Sending: %s", ws.serverName, response)
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error writing message: %v", ws.serverName, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ws *WebSocketServer) Start(address string) {
|
||||
http.HandleFunc("/", ws.handleConnections())
|
||||
log.Printf("[%s] Starting ws server on address: %s", ws.serverName, address)
|
||||
err := http.ListenAndServe(address, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("[%s] Server failed: %v", ws.serverName, err)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"log"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/export"
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
"xrplf/clio/clio_snapshot/internal/parse_args"
|
||||
"xrplf/clio/clio_snapshot/internal/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -18,6 +20,18 @@ func main() {
|
||||
|
||||
} else if args.ExportMode == "delta" {
|
||||
export.ExportFromDeltaLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
|
||||
} else if args.ServerMode {
|
||||
server.StartServer(args.GrpcServer, args.WsServer, args.Path)
|
||||
} else if args.ShowRange {
|
||||
ledgers := ledgers.NewLedgersHouse(args.Path)
|
||||
if !ledgers.IsExist() {
|
||||
log.Fatalf("Invalid snapshot folder: %s", args.Path)
|
||||
}
|
||||
startSeq, endSeq, err := ledgers.GetRange()
|
||||
if err == nil {
|
||||
log.Printf("Snapshot range: %d-%d", startSeq, endSeq)
|
||||
} else {
|
||||
log.Fatalf("Failed to get snapshot range: %v", err)
|
||||
}
|
||||
}
|
||||
//TODO: Implement server mode
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user