feat: Snapshot import feature (#1970)

Implement snapshot import cmd
`clio_snapshot --server --grpc_server 0.0.0.0:12345 --path
<snapshot_path>`

Implement snapshot range cmd
`./clio_snapshot --range --path <snapshot_path>`

Add
LedgerHouses: It is responsible for reading/writing snapshot data
Server: Start grpc server and ws server

tools/snapshot/internal/server/grpc_server.go

@@ -0,0 +1,41 @@
+package server
+
+import (
+	"context"
+	"fmt"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+)
+
+// create a server implement the xrpl rpc v1 server interface
+type Server struct {
+	pb.XRPLedgerAPIServiceServer
+	ledgersHouse *ledgers.LedgersHouse
+}
+
+func (s *Server) GetLedger(ctx context.Context, req *pb.GetLedgerRequest) (*pb.GetLedgerResponse, error) {
+	return s.ledgersHouse.ReadLedgerDeltaData(req.GetLedger().GetSequence())
+}
+
+func (s *Server) GetLedgerData(ctx context.Context, req *pb.GetLedgerDataRequest) (*pb.GetLedgerDataResponse, error) {
+	marker := req.GetMarker()
+	if marker == nil {
+		marker = make([]byte, 32)
+	}
+	return s.ledgersHouse.ReadLedgerData(req.GetLedger().GetSequence(), marker)
+}
+
+func (s *Server) GetLedgerDiff(ctx context.Context, req *pb.GetLedgerDiffRequest) (*pb.GetLedgerDiffResponse, error) {
+	return nil, fmt.Errorf("GetLedgerDiff not supported")
+}
+
+func (s *Server) GetLedgerEntry(ctx context.Context, req *pb.GetLedgerEntryRequest) (*pb.GetLedgerEntryResponse, error) {
+	return nil, fmt.Errorf("GetLedgerEntry not supported")
+}
+
+func newServer(path string) *Server {
+	s := &Server{}
+	s.ledgersHouse = ledgers.NewLedgersHouse(path)
+	return s
+}

tools/snapshot/internal/server/grpc_server_test.go

@@ -0,0 +1,88 @@
+package server
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+)
+
+func TestUnavaibleMethods(t *testing.T) {
+	srv := newServer("testdata")
+
+	req := &pb.GetLedgerDiffRequest{}
+	_, err := srv.GetLedgerDiff(context.Background(), req)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "GetLedgerDiff not supported")
+
+	req2 := &pb.GetLedgerEntryRequest{}
+	_, err = srv.GetLedgerEntry(context.Background(), req2)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "GetLedgerEntry not supported")
+}
+
+func TestWhenPathIsInvalid(t *testing.T) {
+	srv := newServer("testdata")
+
+	req := &pb.GetLedgerRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 2,
+			},
+		},
+	}
+
+	_, err := srv.GetLedger(context.Background(), req)
+
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "open testdata/ledger_diff_0/2.dat: no such file or directory")
+
+	req2 := &pb.GetLedgerDataRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 2,
+			},
+		},
+	}
+	_, err = srv.GetLedgerData(context.Background(), req2)
+	assert.Error(t, err)
+	assert.Equal(t, err.Error(), "open testdata/ledger_data_2/marker_0000000000000000000000000000000000000000000000000000000000000000/0000000000000000000000000000000000000000000000000000000000000000.dat: no such file or directory")
+}
+
+func TestWhenPathIsValid(t *testing.T) {
+	srv := newServer("testdata")
+	ledger := ledgers.NewLedgersHouse("testdata")
+	defer os.RemoveAll("testdata")
+
+	marker := [32]byte{}
+	ledger.WriteLedgerData(1, marker[:], &pb.GetLedgerDataResponse{})
+	ledger.WriteLedgerDeltaData(1, &pb.GetLedgerResponse{})
+
+	req := &pb.GetLedgerRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 1,
+			},
+		},
+	}
+	res, err := srv.GetLedger(context.Background(), req)
+	assert.NoError(t, err)
+	assert.NotNil(t, res)
+
+	req2 := &pb.GetLedgerDataRequest{
+		Ledger: &pb.LedgerSpecifier{
+			Ledger: &pb.LedgerSpecifier_Sequence{
+				Sequence: 1,
+			},
+		},
+	}
+	res2, err := srv.GetLedgerData(context.Background(), req2)
+	assert.NoError(t, err)
+	assert.NotNil(t, res2)
+}

tools/snapshot/internal/server/server.go

@@ -0,0 +1,46 @@
+package server
+
+import (
+	"fmt"
+	"log"
+	"net"
+
+	"xrplf/clio/clio_snapshot/internal/ledgers"
+	pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
+
+	"google.golang.org/grpc"
+)
+
+func StartServer(grpcServerAddr string, wsServerAddr string, path string) {
+	ledgersHouse := ledgers.NewLedgersHouse(path)
+
+	if !ledgersHouse.IsExist() {
+		log.Fatalf("Can't start server againist invalid snapshot folder: %s", path)
+	}
+
+	startSeq, endSeq, err := ledgersHouse.GetRange()
+	if err != nil {
+		log.Fatalf("Failed to get range: %v", err)
+	}
+
+	lis, err := net.Listen("tcp", grpcServerAddr)
+
+	if err != nil {
+		log.Fatalf("Failed to listen: %v", err)
+	}
+
+	grpcServer := grpc.NewServer()
+	pb.RegisterXRPLedgerAPIServiceServer(grpcServer, newServer(path))
+	log.Print("Starting server...")
+	go grpcServer.Serve(lis)
+
+	wsServer := NewWebSocketServer("Snapshot Server", func(message string) string {
+		//mimic the response of the ledger stream
+		ledgerStreamReply := fmt.Sprintf("{\"fee_base\":10,\"ledger_hash\":\"A320C67DA7D1250A577AC5AACDF06ADC25E0EEEF7AE5B8D63CE2E1CC7F76A438\",\"ledger_index\":%d,\"ledger_time\":792853443,\"reserve_base\":1000000,\"reserve_inc\":200000,\"txn_count\":0,\"type\":\"ledgerClosed\",\"validated_ledgers\":\"%d-%d\"}",
+			endSeq, startSeq, endSeq)
+		return ledgerStreamReply
+	})
+	wsServer.Start(wsServerAddr)
+
+	select {}
+}

tools/snapshot/internal/server/ws_server.go

@@ -0,0 +1,64 @@
+package server
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/gorilla/websocket"
+)
+
+type WebSocketServer struct {
+	serverName string
+	callback   func(message string) string
+	upgrader   websocket.Upgrader
+}
+
+func NewWebSocketServer(serverName string, callback func(message string) string) *WebSocketServer {
+	return &WebSocketServer{
+		serverName: serverName,
+		callback:   callback,
+		upgrader: websocket.Upgrader{
+			CheckOrigin: func(r *http.Request) bool { return true }, // Allow all connections
+		},
+	}
+}
+
+func (ws *WebSocketServer) handleConnections() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		conn, err := ws.upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			log.Printf("[%s] Error upgrading to WebSocket: %v", ws.serverName, err)
+			return
+		}
+		defer conn.Close()
+
+		log.Printf("[%s] New WebSocket connection established", ws.serverName)
+
+		for {
+			_, msg, err := conn.ReadMessage()
+			if err != nil {
+				log.Printf("[%s] Error reading message: %v", ws.serverName, err)
+				break
+			}
+			log.Printf("[%s] Received: %s", ws.serverName, msg)
+
+			response := ws.callback(string(msg))
+
+			err = conn.WriteMessage(websocket.TextMessage, []byte(response))
+			log.Printf("[%s] Sending: %s", ws.serverName, response)
+			if err != nil {
+				log.Printf("[%s] Error writing message: %v", ws.serverName, err)
+				break
+			}
+		}
+	}
+}
+
+func (ws *WebSocketServer) Start(address string) {
+	http.HandleFunc("/", ws.handleConnections())
+	log.Printf("[%s] Starting ws server on address: %s", ws.serverName, address)
+	err := http.ListenAndServe(address, nil)
+	if err != nil {
+		log.Fatalf("[%s] Server failed: %v", ws.serverName, err)
+	}
+}