feat: Snapshot import feature (#1970)

Implement snapshot import cmd
`clio_snapshot --server --grpc_server 0.0.0.0:12345 --path
<snapshot_path>`

Implement snapshot range cmd
`./clio_snapshot --range --path <snapshot_path>`

Add
LedgerHouses: It is responsible for reading/writing snapshot data
Server: Start grpc server and ws server

tools/snapshot/CMakeLists.txt

@@ -6,6 +6,7 @@ set(GO_SOURCE_DIR "${CMAKE_SOURCE_DIR}/tools/snapshot")
 set(PROTO_INC_DIR "${xrpl_PACKAGE_FOLDER_RELEASE}/include/xrpl/proto")
 set(PROTO_SOURCE_DIR "${PROTO_INC_DIR}/org/xrpl/rpc/v1/")
 set(GO_OUTPUT "${CMAKE_BINARY_DIR}/clio_snapshot")
+file(GLOB_RECURSE GO_SOURCES ${GO_SOURCE_DIR}/*.go)
 
 set(PROTO_FILES
     ${PROTO_SOURCE_DIR}/xrp_ledger.proto ${PROTO_SOURCE_DIR}/ledger.proto ${PROTO_SOURCE_DIR}/get_ledger.proto
@@ -34,7 +35,7 @@ endforeach()
 foreach (proto ${PROTO_FILES})
   get_filename_component(proto_name ${proto} NAME_WE)
   add_custom_command(
-    OUTPUT ${GO_SOURCE_DIR}/${proto_name}.pb.go
+    OUTPUT ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go
     COMMAND
       protoc ${GO_OPTS} ${GRPC_OPTS} 
       --go-grpc_out=${GO_SOURCE_DIR} -I${PROTO_INC_DIR} ${proto} --plugin=${GOPATH_VALUE}/bin/protoc-gen-go
@@ -44,7 +45,7 @@ foreach (proto ${PROTO_FILES})
     VERBATIM
   )
 
-  list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${proto_name}.pb.go)
+  list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go)
 endforeach ()
 
 add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
@@ -52,15 +53,16 @@ add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
 add_custom_target(run_go_tests
     COMMAND go test ./...
     WORKING_DIRECTORY ${GO_SOURCE_DIR}
+    DEPENDS ${GENERATED_GO_FILES}
     COMMENT "Running clio_snapshot unittests"
     VERBATIM
-    DEPENDS ${GENERATED_GO_FILES}
 )
 
 add_custom_command(
   OUTPUT ${GO_OUTPUT}
   COMMAND ${GO_EXECUTABLE} build -o ${GO_OUTPUT} ${GO_SOURCE_DIR}
   WORKING_DIRECTORY ${GO_SOURCE_DIR}
+  DEPENDS ${GO_SOURCES}
   COMMENT "Building clio_snapshot"
   VERBATIM
 )

tools/snapshot/go.mod

@@ -6,6 +6,7 @@ toolchain go1.22.11
 
 require (
 	github.com/golang/mock v1.6.0
+	github.com/gorilla/websocket v1.5.3
 	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
 	google.golang.org/grpc v1.69.4

tools/snapshot/go.sum

@@ -12,6 +12,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=

tools/snapshot/internal/export/export.go

@@ -4,20 +4,20 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
-	"path/filepath"
 	"sync"
 	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
 
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/protobuf/proto"
 )
 
 const (
-	deltaDataFolderDiv = 10000
 	grpcUser             = "clio-snapshot"
 	markerNum            = 16
+	maxConcurrency       = 256
+	firstAvailableLedger = 32570
 )
 
 type gRPCClient struct {
@@ -44,7 +44,25 @@ func createGRPCClient(serverAddr string) (*gRPCClient, error) {
 	}, nil
 }
 
-func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
+func getLedgerDeltaDataInParallel(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, ledgersHouse *ledgers.LedgersHouse) {
+	sem := make(chan struct{}, maxConcurrency)
+	var wg sync.WaitGroup
+	for i := startSeq; i <= endSeq; i++ {
+		wg.Add(1)
+		sem <- struct{}{}
+
+		go func(seq uint32) {
+			defer wg.Done()
+			log.Printf("Process delta sequence: %d\n", seq)
+			getLedgerDeltaData(client, seq, ledgersHouse)
+			<-sem
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgersHouse *ledgers.LedgersHouse) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -56,49 +74,27 @@ func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path st
 	}
 	request.Ledger = ledger
 	request.User = grpcUser
-	request.GetObjectNeighbors = true
 	request.Transactions = true
 	request.Expand = true
-	request.GetObjects = true
+
+	// The first available ledger doesn't have diff data
+	request.GetObjects = firstAvailableLedger != seq
+	request.GetObjectNeighbors = firstAvailableLedger != seq
 
 	response, err := client.GetLedger(ctx, &request)
 
 	if err != nil {
-		log.Fatalf("Error getting ledger data: %v", err)
+		log.Fatalf("Error getting ledger delta data: %v - seq: %d", err, seq)
 	}
 
-	saveLedgerDeltaData(seq, response, path)
+	err = ledgersHouse.WriteLedgerDeltaData(seq, response)
+	if err != nil {
+		log.Fatalf("Error writing ledger delta data: %v", err)
+	}
 
 	log.Printf("Processing delta sequence: %d\n", seq)
 }
 
-func roundDown(n uint32, roundTo uint32) uint32 {
-	if roundTo == 0 {
-		return n
-	}
-	return n - (n % roundTo)
-}
-
-func saveLedgerDeltaData(seq uint32, response *pb.GetLedgerResponse, path string) {
-	subPath := filepath.Join(path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
-	err := os.MkdirAll(subPath, os.ModePerm)
-	if err != nil {
-		log.Fatalf("Error creating directory: %v", err)
-	}
-
-	protoData, err := proto.Marshal(response)
-	if err != nil {
-		log.Fatalf("Error marshalling data: %v", err)
-	}
-
-	filePath := filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
-
-	err = os.WriteFile(filePath, protoData, 0644)
-	if err != nil {
-		log.Fatalf("failed to write file: %v", err)
-	}
-}
-
 func generateMarkers(markerNum uint32) [][32]byte {
 	var byteArray [32]byte
 
@@ -114,19 +110,7 @@ func generateMarkers(markerNum uint32) [][32]byte {
 	return byteArrayList
 }
 
-func saveLedgerData(path string, data *pb.GetLedgerDataResponse) {
+func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, ledgerHouse *ledgers.LedgersHouse) {
-	protoData, err := proto.Marshal(data)
-	if err != nil {
-		log.Fatalf("Error marshalling data: %v", err)
-	}
-
-	err = os.WriteFile(path, protoData, 0644)
-	if err != nil {
-		log.Fatalf("failed to write file: %v", err)
-	}
-}
-
-func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, path string) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -143,25 +127,22 @@ func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byt
 	}
 	request.User = grpcUser
 
-	subPath := filepath.Join(path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%x", marker))
-	err := os.MkdirAll(subPath, os.ModePerm)
-	if err != nil {
-		log.Fatalf("Error creating directory: %v", err)
-	}
-
 	for request.Marker != nil {
 		res, err := client.GetLedgerData(ctx, &request)
 		if err != nil {
 			log.Fatalf("Error getting ledger data: %v", err)
 		}
 
-		filePath := filepath.Join(subPath, fmt.Sprintf("%x.dat", request.Marker))
+		err = ledgerHouse.WriteLedgerData(seq, request.Marker, res)
-		saveLedgerData(filePath, res)
+		if err != nil {
+			log.Fatalf("Error writing ledger data: %v", err)
+		}
+		log.Printf("Saving ledger data %x", request.Marker)
 		request.Marker = res.Marker
 	}
 }
 
-func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
+func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHouse *ledgers.LedgersHouse) {
 	log.Printf("Processing full sequence: %d\n", seq)
 
 	markers := generateMarkers(markerNum)
@@ -176,11 +157,9 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
 			end = markers[i+1][:]
 		}
 
-		fmt.Printf("Got ledger data marker: %x-%x\n", marker, end)
-
 		go func() {
 			defer wg.Done()
-			getLedgerData(client, seq, marker[:], end, path)
+			getLedgerData(client, seq, marker[:], end, ledgerHouse)
 		}()
 
 	}
@@ -188,19 +167,7 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
 	wg.Wait()
 }
 
-func checkPath(path string) {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		// Create the directory if it doesn't exist
-		err := os.MkdirAll(path, os.ModePerm)
-		if err != nil {
-			log.Fatalf("Error creating directory: %v", err)
-		}
-	}
-}
-
 func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
-	checkPath(path)
-
 	client, err := createGRPCClient(grpcServer)
 	if err != nil {
 		log.Fatalf("Error creating gRPC client: %v", err)
@@ -212,20 +179,21 @@ func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, pat
 }
 
 func exportFromFullLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
+	ledgersHouse := ledgers.NewLedgersHouse(path)
 
-	getLedgerFullData(client, startSeq, path)
+	getLedgerFullData(client, startSeq, ledgersHouse)
 
-	//We need to fetch the ledger header and txs for startSeq as well
+	getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
-	for i := startSeq; i <= endSeq; i++ {
+
-		getLedgerDeltaData(client, i, path)
+	err := ledgersHouse.SetRange(startSeq, endSeq)
+	if err != nil {
+		log.Fatalf("Error writing range: %v", err)
 	}
 
 	log.Printf("Exporting from full ledger: %d to %d at path %s\n", startSeq, endSeq, path)
 }
 
 func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
-	checkPath(path)
-
 	client, err := createGRPCClient(grpcServer)
 	if err != nil {
 		log.Fatalf("Error creating gRPC client: %v", err)
@@ -237,8 +205,27 @@ func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, pa
 }
 
 func exportFromDeltaLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
-	for i := startSeq; i <= endSeq; i++ {
+	ledgersHouse := ledgers.NewLedgersHouse(path)
-		getLedgerDeltaData(client, i, path)
+
+	_, oldEnd, err := ledgersHouse.GetRange()
+	if err != nil {
+		log.Fatalf("Can't find existing snapshot to extend: %v", err)
+	}
+
+	if oldEnd < startSeq-1 {
+		log.Fatalf("Missing delta ledger from %d to %d", oldEnd, startSeq)
+	}
+
+	if oldEnd >= endSeq {
+		log.Fatalf("The snapshot already contains the requested delta ledger")
+	}
+
+	getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
+
+	err = ledgersHouse.AppendDeltaLedger(startSeq, endSeq)
+
+	if err != nil {
+		log.Fatalf("Error writing new range: %v", err)
 	}
 
 	log.Printf("Exporting from ledger: %d to %d at path %s\n", startSeq, endSeq, path)

tools/snapshot/internal/export/export_test.go

@@ -1,26 +1,48 @@
 package export
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
+	"xrplf/clio/clio_snapshot/internal/ledgers"
 	"xrplf/clio/clio_snapshot/mocks"
 	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
 
 	"github.com/golang/mock/gomock"
 )
 
+// Matcher used to verify the GetLedgerRequest parameters
+type LedgerRequestMatcher struct {
+	expectedObjects   bool
+	expectedNeighbors bool
+}
+
+func (m LedgerRequestMatcher) Matches(x interface{}) bool {
+	req, ok := x.(*pb.GetLedgerRequest)
+	return ok && req.GetObjects == m.expectedObjects && req.GetObjectNeighbors == m.expectedNeighbors
+}
+
+func (m LedgerRequestMatcher) String() string {
+	return fmt.Sprintf("LedgerRequest with objects=%v neighbors=%v", m.expectedObjects, m.expectedNeighbors)
+}
+
+func matchObjectsEquals(objects bool, neighbors bool) gomock.Matcher {
+	return LedgerRequestMatcher{objects, neighbors}
+}
+
 func TestExportDeltaLedgerData(t *testing.T) {
 	tests := []struct {
 		name     string
 		startSeq uint32
 		endSeq   uint32
 	}{
-		{"OneSeq", 1, 1},
+		{"OneSeq", 700000, 700000},
-		{"MultipleSeq", 1, 20},
+		{"MultipleSeq", 700000, 700019},
-		{"EndSeqLessThanStartSeq", 20, 1},
+		{"FirstAvailableLedger", firstAvailableLedger, firstAvailableLedger},
+		{"FirstAvailableLedgerMultipleSeq", firstAvailableLedger, firstAvailableLedger + 2},
 	}
 
 	for _, tt := range tests {
@@ -37,15 +59,26 @@ func TestExportDeltaLedgerData(t *testing.T) {
 				times = 0
 			}
 
-			mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(int(times))
+			if tt.startSeq == firstAvailableLedger {
+				mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(false, false)).Return(mockResponse, nil).Times(1)
+				mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times) - 1)
+			} else {
+				mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times))
+			}
+
+			os.MkdirAll("test", os.ModePerm)
+
+			manifest := ledgers.NewManifest("test")
+			manifest.SetLedgerRange(tt.startSeq-1, tt.startSeq-1)
 
 			defer os.RemoveAll("test")
 
 			exportFromDeltaLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
 
-			_, err := os.Stat("test")
+			seq1, seq2, err := manifest.Read()
-
+			assert.Nil(t, err)
-			assert.Equal(t, os.IsNotExist(err), tt.endSeq < tt.startSeq)
+			assert.Equal(t, tt.startSeq-1, seq1)
+			assert.Equal(t, tt.endSeq, seq2)
 		})
 	}
 }
@@ -92,26 +125,6 @@ func TestExportFullLedgerData(t *testing.T) {
 	}
 }
 
-func TestRoundDown(t *testing.T) {
-	tests := []struct {
-		name string
-		in1  uint32
-		in2  uint32
-		out  uint32
-	}{
-		{"RoundDownToZero", 10, 0, 10},
-		{"RoundDown12To10", 12, 10, 10},
-		{"RoundDownToOne", 13, 1, 13},
-		{"RoundDown100", 103, 100, 100},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
-		})
-	}
-}
-
 func TestGenerateMarkers(t *testing.T) {
 	tests := []struct {
 		name string
@@ -132,22 +145,3 @@ func TestGenerateMarkers(t *testing.T) {
 		})
 	}
 }
-
-func TestCheckPath(t *testing.T) {
-	tests := []struct {
-		name string
-		path string
-	}{
-		{"Path", "test"},
-		{"NestedPath", "test/test"}}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			checkPath(tt.path)
-			defer os.RemoveAll(tt.path)
-			_, err := os.Stat(tt.path)
-			assert.False(t, os.IsNotExist(err))
-		})
-	}
-
-}

tools/snapshot/internal/ledgers/ledgers.go

@@ -0,0 +1,133 @@
+package ledgers
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"google.golang.org/protobuf/proto"
+
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+)
+
+const deltaDataFolderDiv = 10000
+const readWritePerm = 0644
+
+func convertInnerMarkerToMarker(in []byte) []byte {
+	if in == nil {
+		return nil
+	}
+	out := make([]byte, len(in))
+	out[0] = in[0] & 0xf0
+	return out
+}
+
+func checkPath(path string) error {
+	dir := filepath.Dir(path)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		// Create the directory if it doesn't exist
+		err := os.MkdirAll(dir, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("Error creating directory: %s,%v", path, err)
+		}
+	}
+	return nil
+}
+
+func roundDown(n uint32, roundTo uint32) uint32 {
+	if roundTo == 0 {
+		return n
+	}
+	return n - (n % roundTo)
+}
+
+type LedgersHouse struct {
+	path     string
+	manifest *Manifest
+}
+
+func NewLedgersHouse(path string) *LedgersHouse {
+	return &LedgersHouse{path: path, manifest: NewManifest(path)}
+}
+
+func (lh *LedgersHouse) deltaDataPath(seq uint32) string {
+	subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
+	return filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
+}
+
+func (lh *LedgersHouse) fullDataPath(seq uint32, marker string, innerMarker string) string {
+	subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%s", marker))
+	return filepath.Join(subPath, fmt.Sprintf("%s.dat", innerMarker))
+}
+
+func (lh *LedgersHouse) ReadLedgerDeltaData(seq uint32) (*pb.GetLedgerResponse, error) {
+	path := lh.deltaDataPath(seq)
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var ledger pb.GetLedgerResponse
+	if err := proto.Unmarshal(data, &ledger); err != nil {
+		return nil, err
+	}
+	return &ledger, nil
+}
+
+func (lh *LedgersHouse) WriteLedgerDeltaData(seq uint32, data *pb.GetLedgerResponse) error {
+	path := lh.deltaDataPath(seq)
+	err := checkPath(path)
+	if err != nil {
+		return err
+	}
+
+	dataBytes, err := proto.Marshal(data)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, dataBytes, readWritePerm)
+}
+
+func (lh *LedgersHouse) ReadLedgerData(seq uint32, innerMarker []byte) (*pb.GetLedgerDataResponse, error) {
+	marker := convertInnerMarkerToMarker(innerMarker)
+	path := lh.fullDataPath(seq, fmt.Sprintf("%x", marker), fmt.Sprintf("%x", innerMarker))
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var ledger pb.GetLedgerDataResponse
+	if err := proto.Unmarshal(data, &ledger); err != nil {
+		return nil, err
+	}
+	return &ledger, nil
+}
+
+func (lh *LedgersHouse) WriteLedgerData(seq uint32, innerMarker []byte, data *pb.GetLedgerDataResponse) error {
+	path := lh.fullDataPath(seq, fmt.Sprintf("%x", convertInnerMarkerToMarker(innerMarker)), fmt.Sprintf("%x", innerMarker))
+	err := checkPath(path)
+	if err != nil {
+		return err
+	}
+
+	dataBytes, err := proto.Marshal(data)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, dataBytes, readWritePerm)
+}
+
+func (lh *LedgersHouse) SetRange(startSeq uint32, endSeq uint32) error {
+	return lh.manifest.SetLedgerRange(startSeq, endSeq)
+}
+
+func (lh *LedgersHouse) AppendDeltaLedger(startSeq uint32, endSeq uint32) error {
+	return lh.manifest.AppendDeltaLedger(startSeq, endSeq)
+}
+
+func (lh *LedgersHouse) IsExist() bool {
+	return lh.manifest.IsExist()
+}
+
+func (lh *LedgersHouse) GetRange() (uint32, uint32, error) {
+	return lh.manifest.Read()
+}

tools/snapshot/internal/ledgers/ledgers_test.go

@@ -0,0 +1,188 @@
+package ledgers
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckPath(t *testing.T) {
+	tests := []struct {
+		name string
+		path string
+	}{
+		{"Path", "test/d.dat"},
+		{"NestedPath", "test/test/d.dat"}}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := checkPath(tt.path)
+			assert.NoError(t, err)
+			dir := filepath.Dir(tt.path)
+			defer os.RemoveAll("test")
+			_, err = os.Stat(dir)
+			assert.False(t, os.IsNotExist(err))
+		})
+	}
+
+}
+
+func TestRoundDown(t *testing.T) {
+	tests := []struct {
+		name string
+		in1  uint32
+		in2  uint32
+		out  uint32
+	}{
+		{"RoundDownToZero", 10, 0, 10},
+		{"RoundDown12To10", 12, 10, 10},
+		{"RoundDownToOne", 13, 1, 13},
+		{"RoundDown100", 103, 100, 100},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
+		})
+	}
+}
+
+func TestConvertInnerMarkerToMarker(t *testing.T) {
+
+	tests := []struct {
+		name string
+		in   []byte
+		out  []byte
+	}{
+		{"SingleByte", []byte{0x01}, []byte{0x00}},
+		{"MultipleBytes", []byte{0x01, 0x02, 0x03}, []byte{0x00, 0x00, 0x00}},
+		{"MultipleBytes2", []byte{0xf1, 0x02, 0x03}, []byte{0xf0, 0x00, 0x00}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, convertInnerMarkerToMarker(tt.in), tt.out)
+		})
+	}
+}
+
+func TestLedgersHouseGetDeltaPath(t *testing.T) {
+	lh := NewLedgersHouse("testdata")
+	assert.Equal(t, lh.deltaDataPath(12345), "testdata/ledger_diff_10000/12345.dat")
+
+	assert.Equal(t, lh.deltaDataPath(3), "testdata/ledger_diff_0/3.dat")
+
+	assert.Equal(t, lh.deltaDataPath(0), "testdata/ledger_diff_0/0.dat")
+}
+
+func TestLedgersHouseGetFullDataPath(t *testing.T) {
+	lh := NewLedgersHouse("testdata")
+	assert.Equal(t, lh.fullDataPath(12345, "fffff", "ababab"), "testdata/ledger_data_12345/marker_fffff/ababab.dat")
+}
+
+func TestLedgerHouseLedgerDeltaData(t *testing.T) {
+	defer os.RemoveAll("testdata")
+	lh := NewLedgersHouse("testdata")
+	data, err := lh.ReadLedgerDeltaData(12345)
+	assert.True(t, data == nil)
+	assert.True(t, err != nil)
+
+	lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
+	data, err = lh.ReadLedgerDeltaData(12345)
+	assert.True(t, data != nil)
+	assert.True(t, err == nil)
+}
+
+func TestLedgerHouseInvalidLedgerDeltaPath(t *testing.T) {
+	lh := NewLedgersHouse("/etc")
+	data, err := lh.ReadLedgerDeltaData(12345)
+	assert.True(t, data == nil)
+	assert.True(t, err != nil)
+
+	err = lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
+	assert.True(t, err != nil)
+}
+
+func TestLedgerHouseLedgerData(t *testing.T) {
+	defer os.RemoveAll("testdata")
+	lh := NewLedgersHouse("testdata")
+	data, err := lh.ReadLedgerData(12345, []byte{0x01})
+	assert.True(t, data == nil)
+	assert.True(t, err != nil)
+
+	lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
+	data, err = lh.ReadLedgerData(12345, []byte{0x01})
+	assert.True(t, data != nil)
+	assert.True(t, err == nil)
+}
+
+func TestLedgerHouseInvalidLedgerDataPath(t *testing.T) {
+	lh := NewLedgersHouse("/etc")
+	data, err := lh.ReadLedgerData(12345, []byte{0x01})
+	assert.True(t, data == nil)
+	assert.True(t, err != nil)
+
+	err = lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
+	assert.True(t, err != nil)
+}
+
+func TestLedgersHouseManifest(t *testing.T) {
+
+	defer os.RemoveAll("testdata")
+
+	lh := NewLedgersHouse("testdata")
+	startSeq, endSeq, err := lh.GetRange()
+	assert.True(t, err != nil)
+	assert.Equal(t, startSeq, uint32(0))
+	assert.Equal(t, endSeq, uint32(0))
+	assert.False(t, lh.IsExist())
+
+	lh.SetRange(1, 100)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(100))
+
+	lh.AppendDeltaLedger(100, 200)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(200))
+
+	lh.AppendDeltaLedger(201, 300)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(300))
+
+	err = lh.AppendDeltaLedger(201, 100)
+	assert.True(t, err != nil)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(300))
+
+	err = lh.AppendDeltaLedger(302, 350)
+	assert.True(t, err != nil)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(300))
+
+	err = lh.AppendDeltaLedger(0, 350)
+	assert.True(t, err != nil)
+	assert.True(t, lh.IsExist())
+	startSeq, endSeq, err = lh.GetRange()
+	assert.True(t, err == nil)
+	assert.Equal(t, startSeq, uint32(1))
+	assert.Equal(t, endSeq, uint32(300))
+
+}

tools/snapshot/internal/ledgers/manifest.go

@@ -0,0 +1,91 @@
+package ledgers
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+const (
+	fileName = "manifest.txt"
+)
+
+type Manifest struct {
+	folderPath string
+	filePath   string
+}
+
+func NewManifest(folderPath string) *Manifest {
+	return &Manifest{
+		folderPath: folderPath,
+		filePath:   filepath.Join(folderPath, fileName),
+	}
+}
+
+func (fm *Manifest) SetLedgerRange(start uint32, end uint32) error {
+	content := fmt.Sprintf("%d|%d", start, end)
+	return fm.writeToFile(content)
+}
+
+func (fm *Manifest) AppendDeltaLedger(delta1 uint32, delta2 uint32) error {
+	start, end, err := fm.Read()
+	if err != nil {
+		return err
+	}
+	//rewrite the range if new delta can extend the current range continuously
+	if delta1 >= start && (end+1) >= delta1 && delta2 >= delta1 {
+		return fm.SetLedgerRange(start, delta2)
+	}
+
+	return fmt.Errorf("Invalid delta ledger range")
+}
+
+func (fm *Manifest) writeToFile(content string) error {
+	os.MkdirAll(fm.folderPath, os.ModePerm)
+	file, err := os.OpenFile(fm.filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(content)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fm *Manifest) IsExist() bool {
+	_, err := os.Stat(fm.filePath)
+	return !os.IsNotExist(err)
+}
+
+func (fm *Manifest) Read() (uint32, uint32, error) {
+	content, err := os.ReadFile(fm.filePath)
+	if err != nil {
+		return 0, 0, err
+	}
+	if len(content) == 0 {
+		return 0, 0, nil
+	}
+
+	parts := strings.Split(string(content), "|")
+	if len(parts) != 2 {
+		return 0, 0, fmt.Errorf("file content is not in expected format")
+	}
+
+	part1, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 32)
+	if err != nil {
+		return 0, 0, fmt.Errorf("error parsing the first part: %v", err)
+	}
+
+	part2, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 32)
+
+	if err != nil {
+		return 0, 0, fmt.Errorf("error parsing the second part: %v", err)
+	}
+
+	return uint32(part1), uint32(part2), nil
+}

tools/snapshot/internal/ledgers/manifest_test.go

@@ -0,0 +1,41 @@
+package ledgers
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestManifest(t *testing.T) {
+	manifest := NewManifest("testdata")
+	defer os.RemoveAll("testdata")
+
+	assert.False(t, manifest.IsExist())
+	_, _, err := manifest.Read()
+	assert.Error(t, err)
+
+	err = manifest.SetLedgerRange(1, 10)
+	assert.NoError(t, err)
+	err = manifest.AppendDeltaLedger(11, 20)
+	assert.NoError(t, err)
+	assert.True(t, manifest.IsExist())
+	err = manifest.AppendDeltaLedger(22, 30)
+	assert.Error(t, err)
+
+	start, end, err := manifest.Read()
+	assert.NoError(t, err)
+	assert.Equal(t, start, uint32(1))
+	assert.Equal(t, end, uint32(20))
+}
+
+func TestManifestInvalidPath(t *testing.T) {
+	manifest := NewManifest("/")
+
+	assert.False(t, manifest.IsExist())
+	_, _, err := manifest.Read()
+	assert.Error(t, err)
+
+	err = manifest.SetLedgerRange(1, 10)
+	assert.Error(t, err)
+}

tools/snapshot/internal/parse_args/parse_args.go

@@ -13,9 +13,9 @@ type Args struct {
 	EndSeq     uint32
 	Path       string
 	GrpcServer string
+	WsServer   string
 	ServerMode bool
-	GrpcPort   uint32
+	ShowRange  bool
-	WsPort     uint32
 }
 
 func Parse() (*Args, error) {
@@ -27,10 +27,10 @@ func Parse() (*Args, error) {
 	seq := fs.Uint32("start_seq", 0, "Starting sequence number")
 	endSeq := fs.Uint32("end_seq", 0, "Ending sequence number")
 	path := fs.StringP("path", "p", "", "Path to the data")
-	grpcServer := fs.StringP("grpc_server", "g", "localhost:50051", "rippled's gRPC server address")
+	grpcServer := fs.StringP("grpc_server", "g", "0.0.0.0:50051", "rippled's gRPC server address")
+	wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address")
 	serverMode := fs.BoolP("server", "s", false, "Start server mode")
-	grpcPort := fs.Uint32("grpc_port", 0, "Port for gRPC server to listen on")
+	showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot")
-	wsPort := fs.Uint32("ws_port", 0, "Port for WebSocket server to listen on")
 	fs.Parse(os.Args[1:])
 
 	if *serverMode && *exportMode != "" {
@@ -38,22 +38,26 @@ func Parse() (*Args, error) {
 	}
 
 	if *serverMode {
-		if *grpcPort == 0 || *wsPort == 0 || *path == "" {
+		if *grpcServer == "" || *wsServer == "" || *path == "" {
-			return nil, fmt.Errorf("Invalid usage: --grpc_port and --ws_port and --path are required for server mode.")
+			return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.")
 		}
 	} else if *exportMode != "" {
 		if *exportMode == "full" || *exportMode == "delta" {
 			if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
-				return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
+				return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.")
 			}
 		} else {
 			return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
 		}
+	} else if *showRange {
+		if *path == "" {
+			return nil, fmt.Errorf("Invalid usage: --path is required for show range.")
+		}
 	} else {
-		return nil, fmt.Errorf("Invalid usage: --export or --server flag is required.")
+		return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.")
 	}
 
-	return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *serverMode, *grpcPort, *wsPort}, nil
+	return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil
 }
 
 func PrintUsage() {

tools/snapshot/internal/parse_args/parse_args_test.go

@@ -25,6 +25,7 @@ func TestParse(t *testing.T) {
 				EndSeq:     10,
 				Path:       "/data",
 				GrpcServer: "localhost:50051",
+				WsServer:   "0.0.0.0:6006",
 				ServerMode: false,
 			},
 			expectErr: false,
@@ -34,7 +35,7 @@ func TestParse(t *testing.T) {
 			args:       []string{"cmd", "--export=delta", "--start_seq=1"},
 			want:       nil,
 			expectErr:  true,
-			errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
+			errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.",
 		},
 		{
 			name:       "Invalid export mode",
@@ -45,18 +46,24 @@ func TestParse(t *testing.T) {
 		},
 		{
 			name: "Server mode with default grpc server flags",
-			args: []string{"cmd", "--server", "--ws_port=1234", "--grpc_port=22", "--path=/server_data"},
+			args: []string{"cmd", "--server", "--path=/server_data"},
 			want: &Args{
 				ServerMode: true,
-				GrpcPort:   22,
-				WsPort:     1234,
 				StartSeq:   0,
 				EndSeq:     0,
 				Path:       "/server_data",
-				GrpcServer: "localhost:50051",
+				GrpcServer: "0.0.0.0:50051",
+				WsServer:   "0.0.0.0:6006",
 			},
 			expectErr: false,
 		},
+		{
+			name:       "Server mode with empty grpc server flag",
+			args:       []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"},
+			want:       nil,
+			expectErr:  true,
+			errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.",
+		},
 		{
 			name:       "Server and export mode together (error)",
 			args:       []string{"cmd", "--server", "--export=full"},
@@ -64,6 +71,24 @@ func TestParse(t *testing.T) {
 			expectErr:  true,
 			errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
 		},
+		{
+			name:       "Show range without path",
+			args:       []string{"cmd", "--range"},
+			want:       nil,
+			expectErr:  true,
+			errMessage: "Invalid usage: --path is required for show range.",
+		},
+		{
+			name: "Show range",
+			args: []string{"cmd", "--range", "--path=/range_data"},
+			want: &Args{
+				ShowRange:  true,
+				Path:       "/range_data",
+				GrpcServer: "0.0.0.0:50051",
+				WsServer:   "0.0.0.0:6006",
+			},
+			expectErr: false,
+		},
 	}
 
 	for _, tt := range tests {

tools/snapshot/internal/server/grpc_server.go

@@ -0,0 +1,41 @@
+package server
+
+import (
+	"context"
+	"fmt"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+)
+
+// create a server implement the xrpl rpc v1 server interface
+type Server struct {
+	pb.XRPLedgerAPIServiceServer
+	ledgersHouse *ledgers.LedgersHouse
+}
+
+func (s *Server) GetLedger(ctx context.Context, req *pb.GetLedgerRequest) (*pb.GetLedgerResponse, error) {
+	return s.ledgersHouse.ReadLedgerDeltaData(req.GetLedger().GetSequence())
+}
+
+func (s *Server) GetLedgerData(ctx context.Context, req *pb.GetLedgerDataRequest) (*pb.GetLedgerDataResponse, error) {
+	marker := req.GetMarker()
+	if marker == nil {
+		marker = make([]byte, 32)
+	}
+	return s.ledgersHouse.ReadLedgerData(req.GetLedger().GetSequence(), marker)
+}
+
+func (s *Server) GetLedgerDiff(ctx context.Context, req *pb.GetLedgerDiffRequest) (*pb.GetLedgerDiffResponse, error) {
+	return nil, fmt.Errorf("GetLedgerDiff not supported")
+}
+
+func (s *Server) GetLedgerEntry(ctx context.Context, req *pb.GetLedgerEntryRequest) (*pb.GetLedgerEntryResponse, error) {
+	return nil, fmt.Errorf("GetLedgerEntry not supported")
+}
+
+func newServer(path string) *Server {
+	s := &Server{}
+	s.ledgersHouse = ledgers.NewLedgersHouse(path)
+	return s
+}

tools/snapshot/internal/server/grpc_server_test.go

@@ -0,0 +1,88 @@
+package server
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+)
+
+func TestUnavaibleMethods(t *testing.T) {
+	srv := newServer("testdata")
+
+	req := &pb.GetLedgerDiffRequest{}
+	_, err := srv.GetLedgerDiff(context.Background(), req)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "GetLedgerDiff not supported")
+
+	req2 := &pb.GetLedgerEntryRequest{}
+	_, err = srv.GetLedgerEntry(context.Background(), req2)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "GetLedgerEntry not supported")
+}
+
+func TestWhenPathIsInvalid(t *testing.T) {
+	srv := newServer("testdata")
+
+	req := &pb.GetLedgerRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 2,
+			},
+		},
+	}
+
+	_, err := srv.GetLedger(context.Background(), req)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "open testdata/ledger_diff_0/2.dat: no such file or directory")
+
+	req2 := &pb.GetLedgerDataRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 2,
+			},
+		},
+	}
+	_, err = srv.GetLedgerData(context.Background(), req2)
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "open testdata/ledger_data_2/marker_0000000000000000000000000000000000000000000000000000000000000000/0000000000000000000000000000000000000000000000000000000000000000.dat: no such file or directory")
+}
+
+func TestWhenPathIsValid(t *testing.T) {
+	srv := newServer("testdata")
+	ledger := ledgers.NewLedgersHouse("testdata")
+	defer os.RemoveAll("testdata")
+
+	marker := [32]byte{}
+	ledger.WriteLedgerData(1, marker[:], &pb.GetLedgerDataResponse{})
+	ledger.WriteLedgerDeltaData(1, &pb.GetLedgerResponse{})
+
+	req := &pb.GetLedgerRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 1,
+			},
+		},
+	}
+	res, err := srv.GetLedger(context.Background(), req)
+	assert.NoError(t, err)
+	assert.NotNil(t, res)
+
+	req2 := &pb.GetLedgerDataRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 1,
+			},
+		},
+	}
+	res2, err := srv.GetLedgerData(context.Background(), req2)
+	assert.NoError(t, err)
+	assert.NotNil(t, res2)
+}

tools/snapshot/internal/server/server.go

@@ -0,0 +1,46 @@
+package server
+
+import (
+	"fmt"
+	"log"
+	"net"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+
+	"google.golang.org/grpc"
+)
+
+func StartServer(grpcServerAddr string, wsServerAddr string, path string) {
+	ledgersHouse := ledgers.NewLedgersHouse(path)
+
+	if !ledgersHouse.IsExist() {
+		log.Fatalf("Can't start server againist invalid snapshot folder: %s", path)
+	}
+
+	startSeq, endSeq, err := ledgersHouse.GetRange()
+	if err != nil {
+		log.Fatalf("Failed to get range: %v", err)
+	}
+
+	lis, err := net.Listen("tcp", grpcServerAddr)
+
+	if err != nil {
+		log.Fatalf("Failed to listen: %v", err)
+	}
+
+	grpcServer := grpc.NewServer()
+	pb.RegisterXRPLedgerAPIServiceServer(grpcServer, newServer(path))
+	log.Print("Starting server...")
+	go grpcServer.Serve(lis)
+
+	wsServer := NewWebSocketServer("Snapshot Server", func(message string) string {
+		//mimic the response of the ledger stream
+		ledgerStreamReply := fmt.Sprintf("{\"fee_base\":10,\"ledger_hash\":\"A320C67DA7D1250A577AC5AACDF06ADC25E0EEEF7AE5B8D63CE2E1CC7F76A438\",\"ledger_index\":%d,\"ledger_time\":792853443,\"reserve_base\":1000000,\"reserve_inc\":200000,\"txn_count\":0,\"type\":\"ledgerClosed\",\"validated_ledgers\":\"%d-%d\"}",
+			endSeq, startSeq, endSeq)
+		return ledgerStreamReply
+	})
+	wsServer.Start(wsServerAddr)
+
+	select {}
+}

tools/snapshot/internal/server/ws_server.go

@@ -0,0 +1,64 @@
+package server
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/gorilla/websocket"
+)
+
+type WebSocketServer struct {
+	serverName string
+	callback   func(message string) string
+	upgrader   websocket.Upgrader
+}
+
+func NewWebSocketServer(serverName string, callback func(message string) string) *WebSocketServer {
+	return &WebSocketServer{
+		serverName: serverName,
+		callback:   callback,
+		upgrader: websocket.Upgrader{
+			CheckOrigin: func(r *http.Request) bool { return true }, // Allow all connections
+		},
+	}
+}
+
+func (ws *WebSocketServer) handleConnections() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		conn, err := ws.upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			log.Printf("[%s] Error upgrading to WebSocket: %v", ws.serverName, err)
+			return
+		}
+		defer conn.Close()
+
+		log.Printf("[%s] New WebSocket connection established", ws.serverName)
+
+		for {
+			_, msg, err := conn.ReadMessage()
+			if err != nil {
+				log.Printf("[%s] Error reading message: %v", ws.serverName, err)
+				break
+			}
+			log.Printf("[%s] Received: %s", ws.serverName, msg)
+
+			response := ws.callback(string(msg))
+
+			err = conn.WriteMessage(websocket.TextMessage, []byte(response))
+			log.Printf("[%s] Sending: %s", ws.serverName, response)
+			if err != nil {
+				log.Printf("[%s] Error writing message: %v", ws.serverName, err)
+				break
+			}
+		}
+	}
+}
+
+func (ws *WebSocketServer) Start(address string) {
+	http.HandleFunc("/", ws.handleConnections())
+	log.Printf("[%s] Starting ws server on address: %s", ws.serverName, address)
+	err := http.ListenAndServe(address, nil)
+	if err != nil {
+		log.Fatalf("[%s] Server failed: %v", ws.serverName, err)
+	}
+}

tools/snapshot/snapshot.go

@@ -4,7 +4,9 @@ import (
 	"log"
 
 	"xrplf/clio/clio_snapshot/internal/export"
+	"xrplf/clio/clio_snapshot/internal/ledgers"
 	"xrplf/clio/clio_snapshot/internal/parse_args"
+	"xrplf/clio/clio_snapshot/internal/server"
 )
 
 func main() {
@@ -18,6 +20,18 @@ func main() {
 
 	} else if args.ExportMode == "delta" {
 		export.ExportFromDeltaLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
+	} else if args.ServerMode {
+		server.StartServer(args.GrpcServer, args.WsServer, args.Path)
+	} else if args.ShowRange {
+		ledgers := ledgers.NewLedgersHouse(args.Path)
+		if !ledgers.IsExist() {
+			log.Fatalf("Invalid snapshot folder: %s", args.Path)
+		}
+		startSeq, endSeq, err := ledgers.GetRange()
+		if err == nil {
+			log.Printf("Snapshot range: %d-%d", startSeq, endSeq)
+		} else {
+			log.Fatalf("Failed to get snapshot range: %v", err)
+		}
 	}
-	//TODO: Implement server mode
 }