From 1238e96423455552c079bb0d95f5e94d3ee3a782 Mon Sep 17 00:00:00 2001 From: Ravidu Lashan Date: Fri, 13 Dec 2019 10:20:41 +0530 Subject: [PATCH] State synchronization logic (#67) * Added flat buffer state message request * Added state vote * Added state to ledger history and did necessary changes * Completed receiveing state request * State read/write helpers. * Added new fbs schema * Added more state_store helper methods. * Started processing response * Fixed compile errors * Added get file length. * Handled state content response * Statefs code cleanup and fixes. * Completed response handling * Completed changes in handling state response * State sync integration fixes. * Fuse mount waiting logic. * Fixed state syncing issues * state sync fixes * fixes * State sync fixes. * Fixed fs entries retrieval issues. * changed desync logic * Added directory helper functions. * Handled return statemetns from statefs * Fixed state folder deletion. * handled errors from statefs * Working for small files * Got state sync working. * Removed cout. * Fixed catering for stae issue * Fixed block hash map flatbuf issue. * Added expected hash * Added helpers for expected hash comparison. * Improved state req/resp awaiting logic. * Fixes. * Fixes. * Block request ordering fix. * Removed couts * Closed non-closed file descriptors * Minor fixes. * Cluster create script changes. * Fixed reset time off issue. --- .dockerignore | 3 +- CMakeLists.txt | 2 + Dockerfile | 4 +- cluster-create.sh | 2 +- fusermount3 | Bin 0 -> 135016 bytes reset.sh | 12 + src/cons/cons.cpp | 159 +++- src/cons/cons.hpp | 15 +- src/cons/ledger_handler.cpp | 111 +-- src/cons/ledger_handler.hpp | 13 +- src/cons/state_handler.cpp | 339 +++++++++ src/cons/state_handler.hpp | 52 ++ src/fbschema/common_helpers.cpp | 36 +- src/fbschema/common_helpers.hpp | 11 + src/fbschema/ledger_helpers.cpp | 8 +- src/fbschema/ledger_helpers.hpp | 2 +- src/fbschema/ledger_schema.fbs | 1 + src/fbschema/ledger_schema_generated.h | 23 +- src/fbschema/p2pmsg_content.fbs | 47 +- src/fbschema/p2pmsg_content_generated.h | 772 ++++++++++++++++---- src/fbschema/p2pmsg_helpers.cpp | 210 +++++- src/fbschema/p2pmsg_helpers.hpp | 25 + src/main.cpp | 12 +- src/p2p/p2p.cpp | 28 +- src/p2p/p2p.hpp | 46 +- src/p2p/peer_session_handler.cpp | 27 +- src/pchheader.hpp | 1 + src/proc/proc.cpp | 25 +- src/sock/socket_message.cpp | 12 +- src/sock/socket_session.cpp | 3 +- src/statefs/hasher.cpp | 51 +- src/statefs/hasher.hpp | 25 +- src/statefs/hashmap_builder.cpp | 96 ++- src/statefs/hashmap_builder.hpp | 11 +- src/statefs/hashtree_builder.cpp | 71 +- src/statefs/hashtree_builder.hpp | 12 +- src/statefs/state_common.cpp | 3 +- src/statefs/state_common.hpp | 27 +- src/statefs/state_monitor/fusefs.cpp | 27 +- src/statefs/state_monitor/state_monitor.cpp | 185 +++-- src/statefs/state_monitor/state_monitor.hpp | 47 +- src/statefs/state_restore.cpp | 2 +- src/statefs/state_restore.hpp | 4 +- src/statefs/state_store.cpp | 357 +++++++++ src/statefs/state_store.hpp | 35 + src/util.hpp | 2 + test/vm-cluster/setup-hp.sh | 24 +- test/vm-cluster/setup-vm.sh | 2 +- 48 files changed, 2534 insertions(+), 448 deletions(-) create mode 100755 fusermount3 create mode 100755 reset.sh create mode 100644 src/cons/state_handler.cpp create mode 100644 src/cons/state_handler.hpp create mode 100644 src/statefs/state_store.cpp create mode 100644 src/statefs/state_store.hpp diff --git a/.dockerignore b/.dockerignore index dcc16fcd..d4eb2f11 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ **/** !build/hpcore !build/hpstatemon -!libfuse3.so.3 \ No newline at end of file +!libfuse3.so.3 +!fusermount3 diff --git a/CMakeLists.txt b/CMakeLists.txt index c3f17fea..9c7fe8e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ add_library(hpstatefs src/statefs/hashmap_builder.cpp src/statefs/hashtree_builder.cpp src/statefs/state_restore.cpp + src/statefs/state_store.cpp ) target_link_libraries(hpstatefs hpsupport) @@ -91,6 +92,7 @@ target_link_libraries(hpusr hpsupport hpsock hpschema) add_library(hpcons src/cons/cons.cpp src/cons/ledger_handler.cpp + src/cons/state_handler.cpp ) target_link_libraries(hpcons hpsupport hpproc hpp2p hpusr) diff --git a/Dockerfile b/Dockerfile index dfcac61c..c88d5142 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,11 @@ # Otherwise, hpcore itself can run on any docker image like ubuntu or debian without NodeJs. FROM node:10.17.0-buster-slim +# Install fuse. # Copy fuse shared library and register it. COPY ./libfuse3.so.3 /usr/local/lib/ RUN ldconfig -# Install fuse. -RUN apt-get update && apt-get install -y fuse && rm -rf /var/lib/apt/lists/* +COPY ./fusermount3 /usr/local/bin/ # hpcore binary is copied to /hp directory withtin the docker image. WORKDIR /hp diff --git a/cluster-create.sh b/cluster-create.sh index 47bf3726..539b87ee 100755 --- a/cluster-create.sh +++ b/cluster-create.sh @@ -52,7 +52,7 @@ do pubport: ${pubport}, \ roundtime: 1000, \ loglevel: 'debug', \ - loggers:['console'] \ + loggers:['console', 'file'] \ }, null, 2)" > hp.cfg rm tmp.json diff --git a/fusermount3 b/fusermount3 new file mode 100755 index 0000000000000000000000000000000000000000..cba0239e82dfb96ed09ec4f6489f8230781a870a GIT binary patch literal 135016 zcmc${33yaR);E4T-D#4LbcgkZMQ9+bl8_Jr2@p*nKnKI3ETRY`gajfXiAjgeEto`U zn;yfs4CCkw?vA4lQvpjH`nVYK)2Rg*1?cFr?t+8Ey+sL|$ z@Q*Tt36X?ig$UB~@h|gap!dk#r#nA+@9EA!&w1T~X6LG03hr~!YoXy>^nUHg`KcZH z*X_tZza9Es==Gqk?L}M8WlvK(dar87{;S*J*W2MwYKQ-_cI@Yp1_w+o8{B zN6x5r?0KghIiEw02X$?)%iEEY-HzUWL%_M>zq%dzpmyY!wnNWvhd!Yl`qp;z=Cwn= zq#gQo?a(*1L+{ZJJ*OSNJ=6~Wk#_o3&<@?(j{FJj$Y0$Ke`P!JXSG8g)Q|+X@G+w`0%ZcH|#!hrX^IIfL4<=Wg(K(t2pw zl^#gcG;=?a2AFh@qMQ4MI~4zrHvAga_aZLmqqXW{Urp%}t=d;rRJlf5w5YPGwA@#+ zsA$=8t$1~*j}_%bD=M{$N?&QY;HxTHR;5)IS5;M1fxoo6q9`+SQFV1uVR?zRwAfd< zitzMEZ7i;bMfwqD4zrRFo?d7D?7~GGgXLwEACL zT-MrK@t7^FR0lX(+D3-?oeE<8x&Ftw+M3=0duU0qVRwuvteN}yE~TY$n$|&UfGW~- zfjhNZ6n@*rJFuOO)c&e)Uyssg-l^QL@UDX2u5D3xO;6TGXxoAFGVhnnywZ4qq^~aZ z2*9MzRP@?5^jVim{>^RZFU^zmrZ)7~75#7<`a6oQUD;ai2a2BEhJN{6DSuuY`e%y2 zvJL%+qOWg5|5DL6x1mo}{o2-sepK-vXhZ)&(Tz6r?-gBB_nA@BZ0(GqC$ypetmr9i z=o&ZbxV&xXXBB@z8@f~RN3~f8OuVVWBjoX3%(RJw{T8~11a%#-(7BeI*Fg(C$0VXW zY@tuG(9QLV_x!x)GcR+!B7K@kL}{*jq+evAZ(89ISO@%azL?i$3tfe{sCw8!=e?+T zZL!eJHI#MREOhENuO% zL=nEuSm>@W6+pR1V6@RKAo&wzp_{pfHSrdDCkuaqg>Kz_LQk>KHIo1PgtXNkr+f(99be@Tr z*Gvn2oJmAE&q5z>p%+-_>d08sF0;_pRzT1zE%Xa5a#maD*%o@Og+9STUvHsLw9p$Z z^jr&llZ8InLf>qmPqEM+w$Qy6`W6d4&qCj3p-;8Yy>-W;yn)E#c^=K%(Bz9~KH#m} z9knaW(VQMJ*P}HLbl}(BHyaS~WvmY!ZpJn6mpQ<>WQ7h0{3LNMRiP$_*&vjy`cjFuOiNr8)_2xmBg89 zLt6yCnD{8-n+1M3@pR&i0>6YfQ);MI;8Tf@Azmr)3B;K~Lj?jKOPr}QG*jRsi8Ez} zvIRbjI8$XPMd1C3Gew3x0`EzjsWB8U@Xo}U5<{B6V~8^qhEDt{#-DgL@xucDWft%p z;s*qNk~mXgs7c`85N8SuZ4vnA#F_d+o5Sr(8i9)|=hB?HIsUUbb8=?q`G1--%e(H_ zCL{}QpwAL-z%h{H?)BFvt_Ll($?N~rxCk~Ob9w!d7b7s;^&W&`esJJ&c)%M>x(tf* z{QF9h-F@qYd~Yyr=rrWIV7Ecwn3UR-@BcQ}f7+OhzSf;-_ARP2=GL7_bl>znQi``I zw?XsTjON27X8V1SFQflVcSoOMKHL}if{EWR@dr)(Hsbk#fraSs(VN_TYiWRgS0Hi& zIFM`O>arRD!R8eFdW#}cz?hqP!W}$|=@y)6%gsFL-rzx9UZ8A`X50gJH8i;!E~I7t zZ}a`l##zLb=)5D*U7t#NFza!|hA{`c=#IPoJTy0!s+O-IZM?`-U=*XffyfTR5PxoB zS9il+kkNV+BFWHyQ9jL09OABj1lAadN<=a;A=r9?Ew$+drg`IM$byWhx+a^u;cHR` zqkSNwuDOf5J`(Ecnj_rxzab?Vk7Gu8191I%BG13$1^PC% z$(RLJ<8cULm$xDxc?07Ujk|wtZhny*MiHy}{{kFVjlwpWgxq+ks z(BrPh>3}w=;keJ4^E~aPoLv96#^-22lk2OH%!Tl5P!LpsyF_>`@uu|_@#bzg1KYjp zb~6y?FUYwfXF<-woJG6fhA#B=OX$BCgxo}ryS^JtHBLbiMmW+4BC*w7lNi$_eLo$mT;@H;8>)cBa1+g}(!YoI3so#1A{ zj+lYJ>RxIF_2lDT|9NA$pU4CAf!Ya8 z%hKR|cYQVj7(?_lhA6dZdo5bOFadrr>}Y}+mgC>&u0M+9IrS|uKK^~i&(e(b6VcKV zcgAO@MKIU@12T;IM2KbD;y~7Kh!;kkN*B7$gKfq{>Et~BUU&UTEbnF*7RHBiLnueyD52pok z6XVfKFAR8wd$D}~-X9)x|D~z7R^Q~C4}ZOh6)WGrFCT)VjMJhgzKQv07KO@{XtrH8 z+h>eDC1QV&S@{SI-Ol|Qnj7bdN_RssvbL~qT3|_H0z{mFhw7>JxP38#vmpj(KBRiQ!8K8cF0Vg7F)H8xxpB`= z!v75+Y-_v$5GS~|=JtykKin5pj~w$B$oW_aE(e=P2j5G#c{w57oV*@o{0rDkdV^0T zZbqjVcV{r=bNssvP1WCq`kcDm$cEowA4-lRKQ=c{YG`(^AA>ZS`c~?xTqN%|Wjw6U z!RCJN4sk3>Al|FfAj z&x6f5b^B~GS%f};5Kia%Phe6$40#QPFrj{(Ao1hC4X(B#XMwpr^HHA@85`8zKXt^e5hg#lZAm!al64K#G4fV zM88b$zIu^Ds!M7LS;KVUjdXl9qMS8+i|8i&V@-l+GcsvmENkL{X-jsAniSCnH62;w z0k-IS*@mau-fh!oPoAaKlvJ~J2q?3@leGs}8 zIfd{{z~_6>4)8p{TdO2o0C?);S*fHi1AL(40bxxg;M{4FhtEp$=SsL1aA~Q8*8?7f ze89GifN3w`O@L?QNO&{gl$H5wNPie`bzbo$gtq}+R(+M|TNB{hdPE4D_XCdX5t&Gy zgMcT@nmmv2VZb{^>=57^kN64B#p$60c_NHn6udu489zWV&?MriNn}_>SbAvUu8NuPB^8G)-mTM{9 z*Bez=)O?C{;E<&{r}kP+t9hg0kT5RL%{GzhN{G(kO^XhgdJP!!_#RLp$KiGyr?m8*;%hcnLjI5PlB%KiTT~S{m z{Fe3Na0n>*PNm9KQNRpfFsuF)5a`T@cWAm%Aas9i%v0(-#&iV@*$#(ksi%w;kdLLz z48Of9ycyfvI3a_J*xrv)z{qn=q={eyL#xPV7Gy@GgCXEyRP&;+Z79hO@BnGJNZy2! zWHi(;O=$+F6a{Tz34Fc6Nk8gtG@d<( z=y8NT^!q&d{*(Dwc+K$1_n$I4W67p%4Qb;3?>l27goh>~iSRyiTCmv0DQgRTgEil> zpAM#XMTBs|ZNv)4Xnls;iD0@-k=H8ny^{PR{6Ov^MGi>v5k<~dWS<~+`v7D)7k+@p zDE%KD;Kx(4-&1k}Sx-*#U>Dq6GY)e+Hq5<8alei20V?hbS3IgJwxfbOriPACG$|GT zf9t3t4?CN@<@>PUpO<(6#QJ~s2724Q4Ts$Iu8t6dOt`oc8n+(h(B!(e0?9?gYijPvE83CgKM3LbvOIV&*_7b59jsg3 z6`PEL_`tF#cl}SXH0n!t{apZ}kx)V_#4artUABuZyBm&TbeOdn#-Zt`HZDU34DIKD zV=RWTzggo@q#n-?roTTFP3Ivr5DenZZw30zjoK(l^G(PPWXG#5`RP`>a(E&AcQg-g zymx%V9@XqX^SV7Y<0K4%)S4G4t^swKNiQNFx^L-^We*Ey|6Fu3kaY+&Y$sf%SqU#zz;F+(~(iXh5NbqJLRG4nceLyw<1#RZOMD2ncQ> z%~(!-%b0?-PShP1E3HhCyeFT|M!c!oUDFY1uq)mP=a&`q4`&CnZKOlS|G451zSO)STeMt>0y z?uI)sqe4I6$Jk7!x#*ZN7s`2{fpd)4(I`0O2X7$hGkC7<%mwZnS0YszZ`0j@tPMHB zoeuZ`?o5S^IMcxTZ*EGi0bA&H?x3@3(H=`#BC?9{lR*W>pHQ2Rod<@e4z-X_!*K~@ zr9-UcaAY*497VDcgA|%C?r7p>vPmE-TAHv9x*=dP>A?DzEtk1zGU2%E zb9h4)xJ{&2^Pc|w@OGunu;HdMm>bbtn=!nG_x(l}xPVr3J{jVGDwl zGTQR6>%yU3K(5C5{*bqBujl0B(;8mFgr6RC48>_01myeQ$j4lt2Zoc6=iBy*Y;a9# zbDsZo<0V+07rd@7ZtRTb0gI`JyX+LEkcQ*=NCxxV_5Fb%1ikQKHh7q5^9LV|UEg=w;D0iTS)I&PF!k2xM?+_Wx|$ zY)%FqwV0cS{J{NUD2zvsa9p@GxEGMx8bsnQFLaEzHrv-}wNcQR?RDR`(_Occhhljc z2ZTYyg&rEc7Pcb{9!hK*i6_zZkcN#dw5sTKDd)y5Hi~^EUAGDLFq65r!S9D-aqyshK8w!M8sD>R3rq>LBBgB?- z2bMl+YM6i+&BJ$;LOD&74iacTe+YHyZwW!Um$DQcuzPm$}lyI zedq``tbLJNkr%;?yx19tJnana5O?wEn=t>x9a_Hs7npMu%#k;29Uuha(~aFf7+NTdVxgq0wM-`t`F(YM|mq)`33!nWf&*d{xj}+zH46B+`Xp8 z7z7>>jKR1PlBQv_|;{cdRk(}^$HzLBl-kV4~W|sdNSw5J43WDG&d5Y`}&TKZefL~0#`@zi2xE{7) zBWH9#=!R~>ie)6ZutX1K3_baom<;)-g3~6^TY;>znJ9wYrnh$58IJhPNLdKs%@FC2 zd=Px0-Jqy>q}OGFT9na}2#UaIyl_}10tc6#Y1rSWW-Dd0eX`jQOS9o@c9UxMEoL!uY^XEu z>3DaL_1hQ>POw`}pYW&{Kf~%!Pe#x6pNg1CfDDXdCn^bKy)0YQA@w17=J^eCuI*sE ze_=mtOp)ylgTV2JIcWE1+3s&hQkDt2fsIZvKtD$#@l0rqxK)X}S2mk3o5donn2P4H zO&kK*_~a8|;3^Q5C(76=*zGH3p+Ht2G`3{;zHIm{HiM_%?Z>;5H{b@uP|iLlBMOgO z9Gb5$b})Z|H^Lh+%sC{=C2YD0VS`&uCaijB4y^}n^n~5{IB}{pJfa5-#7cMlk77>7 zeMIvES+|cCW*xy2t9p@o+-Ss<481D(o|1ekB;Q;Nq~J5k$nz5HG){gj`WgU1^|e;I z=4Z@as2GW+mcFL3;VU#)?w4mt=3MERNHCl3j3(zna6-uObebDCNHX)ahiU%La$3gt zV?>f%Q=SB5ER;5#2CUQ^lO33clOKrtJH%OR`Um(z<8(+cvfl@7ZLn zp#EUDPhjQDR+@3ZV-%oy>hYY*axM@3mtl7JBjJ%XAVW$Or3A9hhXibbex?DzZU^vO z!eT%l8gLAAT!4soeobn^aGPDazk+D{2@qcV|l6qpKo?T#uo{druUnsHY z*~-?Basv~H`%t(v>sIi?phqnet&v>EkyUg2pN38%1sX2dqlZv|mYZeE%Vf*3NLH4K zGmDM;)4&f72{%;9M$2WRMA;}6UbZyq10Fm|fj+~Ip$lCLrSRG5G`cP>MY@iCfu(n% zFhk@BqFoOAbAZ4aDYd-7hQV&{w}iFvEGNcga=-g2jR<6Il3f3>aNSO>t=PdB0i@4R zHbM|m|1J5KN&a;vzp)hD*hNGc)8U_xlgz{rc`8sO#JB}`y(0X6#)4fu&1%6RvXihYJw}KbiW{o0ygSa?z5Wy49ma)T+}pi* z)|UW*|5O5FDbRdg&{ke;*7so3YDReLvaTmIhxWw6%Xb1ZR-nU%7hTvJ`4Vs&o7ZZ% ze%J+(xsrWBs017m&zU_Z&;Q#jum64j4_^Pf#&aKF4!`8hJXq7s>&F8#anwCn5V7Rn zwqw43Ki(jqFm8<@4^4mfvh8W62;&;Cdi|e>jnGcx+k;{~yse2Z<+-;u&8I!E!s4HG z#}aT<%Ed02H3I^d;K@b!HJK<>VV4zr3X7QyPsV^6A3`SY^jU{@9J&hTnkc;AKta2V zVX&Y@e$J(cls*_+yq1G^ah%Aj8eDgKa92p%W`g@dX-Q37C>^E<&($5bai8-p-7KuY4KL(0 z9IyGx8@M{E_QDaqe&W3yFm16gy+dV^q2>!O&LMGPYkVh$$G@lM&1ns%eDSD{=Sl5B zL9Tg@iek}+qmx_wX>vq#N`fryP58A!{1@P5**L6Wh{ODUJ^bH;R z2JysW;6PSWQrLGmbuxBfSdZRgOnp~4{0m@c^b!0E*4x2~wx5W$JViG1^#-toN|7$W zyG8PrO5O-K5L-s0b}0$2u6&I15=L#p}O@uW#_j zkK;^>_wfQ3Pg!Q;nD#B)lhLWWd>46xzQkB$1FaVKzn8;%g6^*OgT&j~*ql>M*VoKT zZOU19SZl`2xxMK$9Fm_n1Wt{W3FCT1W6Jzp#yDx`I~sFt{EQop-~GS z|G}eM!S?|90$B@D3%5-HyXmm`G-rSF(MIF5w}l7RO3Bq|AwAHsGY0H_;|3@$X%tEG zCV=Qk6N|{PU^L!^Z$SAI0?8klNM)Ei$ekiz1+u!6XCp1o>Yyq^i@{scC|^8$O{HZf zb!O!wFeNW=EezQ0ZD_{B;%*@14*eo{war)yRfy|6#IEluJoNF}zT*RlTj5JQv*@nY z4~BPc=C1W#>`!*CT;nZzIX5@~Z#u=gYk$6?<}*~S>^iNe39qEYc0f(U(br{O#EmfS ztggW47uW;wm`x@CwI0Ejm(huVtj^~PU!IRVD12#jLWkHW70U)+S`#ZP7Nw(4G`8d; z(Y-0?4ZL;YEzuKS9A4W(k9?8BY0z+*q61l5h6xQJWCwG!9z*9-n~uI0&L>8*)u~>Z?%I0sQ zQ1v!noAo%~j|6dSNMIb&gd=#9cz^EwerL@0=k{?zVXEaPChYi5zDdV<7v;{~=M7#R zk?+T=13PhG_o=q?MQpw>a)<2sLw6Y|>?4jk<>nj>ko$Pxh=l{9R=MErAz1k5F9_ZE zCO89e*VEzH|H&SVaJTq99ioh;19%3hVeT+ChG5cQd%*W+^Zf72F#c`MtZDu|dAOrn zcPv9J##ivQDET

yJ&eWD2}$0uCmNPmM7coEJJ#E|3_gO2wY!8*EFAIH(kB`)n{9 z&4?S7I^F>{LI#J3NI-@lW@9w(m|ehe)}R$6f|S|8|>=_p+1~m z{0H%1gch&cO~YHQd9`RIj!dNhFP^qx3kpenu12=o{)cvDYsLQ7*cRPmjVC{AWi=tZ>7XH(CZz6Iaz$-QJ*7cm3sZGszE@A9v^wsUo&PQxU zK`i7j4x2=(&z=~_U@vMu>VzR&$$I3Dp*H-eF&<_%uKMe9*< z^3^I36HqGy@p1Gl9Ej^6I~<7huc<)f2O5l(s8WIWHU=7jxRikyB?57cj9EMitml3c zff#kpKum>51|rqB3d8}lcHM3oZ3#q~pIZW%aTo>pc>m&63`!A35pj4I!*Q3-}%U>8>c#bby-i0*U+RjJVayR4f1RCINLtl}K z!^7akEW1|}c)=8L_%dKTlbeKFB*Y(fg;${U<(K6IE;Rlu^;{(N%>ILg6HqUMEPlj{^qdeE4!!7C zROrowDC1X*vkJWf7+8efl9tdrE2EU5ca)>vUg&Loxt-7x)~?&l!VEom2Y9ZytArHe zD6C`tWB4Kefh6K?Z#eFTp$6mmMSl?q-(fS=dOSab&Eik1)czk5nEya!{*$f#^@mpX zv(+$5+zpWJo|ElrWV`KuXjjH|U$%<7&`UD^$(G|}%fvsl9LbjZME-*+<~!!TGVmeJ zu0nx)H&_(i2B4Vc56!%iSr0+?H3Bs!aDmZRsyNtBxZ*Q#wRXiD;FBTu03hKHZVY*s z^Du_9Rnpo70z^Lt;z=1X`5ny*#wto=}{f2*cLPVIPY zgzpNNJ&gSR&sBUw8A8gpoNH#wyXd|&I78k7_M^2bPaZNlg&9K2Akx?Yypi*Bnt!jj zjr<+HeLv6tqu2j-p8qvtBgPibEaK*;Hifrojnn)m_|V_~p?IMOdnNhj7$L`L4p6ox z7nUn&*GV|BBe-PL$F{G#uh9sJxfNHUvAGL;KYH^^M|o*MCro#4M`?o7i;= zW%KbGjZhXE`m;Fvsb{rjj6-|Ok9X*rKvuA?hbiwxcnCM|cz6I~&Ar7svb0u_ZJz(Y zR(Yp!*YBd!@nqx<*J_767sB?C&#skya^>b9OJTEz4kI)Zuybw|no%I6Hq~B$J8r)G z%KX@kwuti&+z{YlJIMdrzZUWD#n$%CY;CXjbCKYLX6$0!TjQ&`g|eAi`6z#DJZbeQ zz-xaQ;XAq)HR^-_`$;$qs&VduYV1(X+R-X=SAMY4hM@k~1)_V>E6>P5I@K~=-1Q-j zM=cD!SRuLF=l>=|xtTHnna!zAD?d-rb4l?5e@@Rz&cRm5tZ|0~f0 zRPv}PcdzhTO@;9?I0G4zaFkZSwt=i3=Xn@&55%>^TzEelr^FYd5fcGJA)E;Ec{1`J zdJSiPs4yoJ{_-F4fOw2W>lpw7lJoT(Iscw*zz`|NeK}tL7kEwhezZE;UFw(b2^#5gWNWf< z-=qG%3ii{9$Vi9yfb2hvU9ja8-YSiHq*tY?sq{fz?)!#+nM(=fJ}4%n_4>F7z_8CwmGem*t4km)U&4Mw~Z z-$y(S!`?SM%r%{nOycLueS^3GJNk){i#8%(w%jkm6KA`4cvPRna#oK2cicMW5B*AP z=XU&VH{RGm2ORf#QV*Hqk5@4;4ROn!)7;VRILrH9_!(^w2GH_d=(ykr^!Ro!0H%LHL%)KUg7fy=HexiroI!U znn~W_1Enaz2r>M8Ry8D6c=!RT7_DlRW7j?orP#+mq{i8H26o1CkSSW>)l1chrOYO1S7@Z()&BZ_@Rf|e}4 zABD=QMf~p8sznvPWyMvRP>fGat@dPj29^q4_%ha#((C}D@()M1h7dftnpPWswo$`w2~!RT2)1bZ$-rt za0r7+L0Vc>QB%2yP#M1ERm|^mEzt(nBoD07q`hn-?5ioSE-Wclu7#DJ(xv4URU8v^ zL$ni$tMOs3EY)CndBv)7v&G%6yCPiMbV})eNL+ICJe9?W&qme41@paapCN zA`By-l~;(bq?KbFRs!Lt;k3f)HRVM>@YOebtqhSSaQtmmtI4sVuxh#VMYRVXI$K%j zE0*18!?(D!e2J&BunJ$EW5kY7OqhR^htK5|SK$L{9?>J>EAch8C7!~?kXq$?4O1i)^ zuzILe`bWL|Oqz!=UR_yOWcJy#uV`6m8FB}mUk;D3BZy0Wy#24v23SPu5@pnR zOeWJmNozg$)S;)M9R64=y0XNxs?@g(QqAs&ev8<}?68avGD6J?L{Tfg;?=&YLXTNh z?I|pQmpsU7mX>A%`l$lmtoE(JJQb2Tcj(;en##(GDvTF-#kj!9aQVv8B8)W{O7LyJ zCBr?wsx_X$NLpAkU_B@7V9a`aF41gPQ!djC0v;noTM&Z?sY0IzAA>H&*pKxL9>~li z^;q+)Vz8_e^x7Z!Oy86juM$ll8wto$Q6dtMT$Q-itSBsBkOLb7;PRh42|DE&wwtz@7D^|zvVN<}-awUAPX1#D>z78K>4_V|6^3}*>X z#UC4n^)PCXETll`R}a@Z&%gmGsVQZvsUM&4TwJ@~0axG*%Qzn*+r84Bg z;|z}=r3`La{szmgnd=r>E-9`=Dig^CN)S=W14}*2krIl_uuW0%qV#cFRgDwAfQb)Gav>0#D{_k! zP>hWwPB70g)kObfn1~735}ZLi8F&l#CI7GgR{Rfg5}%jyyQuEp_VBjGfuMHSuDnGPSknix<|9+26rHLtt0R&_r%=JsH|ab zSF!&S`%CQfY6g~D#=T{`!YxD%=4*@nQoPEm;e67HPxeAF+Irc?9fNGVU!_HDZEnuM zEzPp+&CLZUSEJlw$A60eZ!7WUXUNCR&8u0DlHVzR80D)dqdsYFKE`^yK$L+yw1@Fx z6aT*WVZ3X28_I(yA4Pcvkv=6@Bv;zJ74jzI5)qhB`W^?Pr+BbS98Tr zb|p-8cU%=!t6kW0+_1F7{$~4J0?)vQEY&pxjL6+gknHq^Bq|DFSHDoQQK6@Mdn>+I89o-14lxvuyeaMJ!Q`1cO@=8|uUYjbT> zL|%+5DyJh*Fa90CzpfZhe!rbMZ;8lrCH%=g!Q}}$Cb))di1fNrZgl3jGU__yyRz+N zwis8&L|00VYY12-xDwC=M&))Kh3366<_2QmdX#PLpW)h4>xj53#^r&XuosPl4IZ%a znPSSP&CO>4voCcKxvuPpH8G+W;G)kNXR(Nh5ln8%}Ysh3_k?h8ljzyqMv`{9|PkhRF2J&8oyeX6i zJqwf*>BFw{9b){ytb?RKQCHAD$>rHVT{lMNx>D+#5sM%@rIsbi%H-Uc?#sV2nVZTClD5Oe-8 zh&dhYy*s%MM(4N=NBcwx-mVdVW`ME?k*ob__qoFHrsPt58EfZwt#*DIyiqe zLuWnm$+iD?ol{+hBO;zYr_L!trxCFPa=k7CbL%kVF|IVs?fa3JE>iN(J=Sou5Y2gF z9rDQ??ddkxL5vUO4S~3sl=pRW^9;mStMP8KMeN0ZBLpy&b2~E5t^x0U-nNFh7*yC zJt6#k6lJ%>Fmls0^ayDTJ%TJ(!W5aO*q?c5GaPMxZ9Q%+d8j3Zr@9V!?CIx@;i;Uj zwBcdM`{-zM^HYG?Kb1?p%%#&^TOuNAt*K~Y$3(Dwgf`2LVeMfXk%y+cJQ0`6JT#e- zG4=7!fwiOwe8nW3Yrct`>1EEG=E|-**IeTnwZ<(+ty#D2;uK!w_#YOaPxcFZK^e~j&)+J zn{<=k{H`|hlX`zdde7`o!{=uJuga;v_zfH=KIP@2|>F*N?PgM1I7Afl6_Dd-H|KO)_x#yfx z?of_+P?cL%xmT5Z1DV$mRi09%Q#qo$DhH`@v??d7a;_>%RJl@>H>&auRX(W7t*YFs z%J)=xM3tvh=~My9zjEL;NR^{iIa!r+Rav6Sm8!f^m3OG}K~-*5`Xy9EZ@8Szro*b}paA{IbkTpB1!`ne^erEtbRI^`ND%ptHGBXcMHx2D z!5jSOTXq9++>2LGqi>^>=%^UHF0MJANA`@qSIE7C<~$@o{v~|$qr!%pNZlepp2J5! zDHuMYh^GZuh5?G+MzCYlLa2)2dwy`t2FQpd!Lfkt_K41XPllXT)XW7SW^$0=op4Xg z%rb%jbS`G`GXy`uPt1xeO4tc2J07GFk+rdr?2zbkYzKnqGHX5pD7x&{9Q-PBY}9|* z^h$tn9atInEo8^LqWB-GbKVNFE9OVGcE*%wCl9jY@d(m=pmmI*3^Bayff!y= zn@}gH!%S)yE&eZv@J{a{N4PJvk>2TJIwD)b=h68&0`3@93?z0ISYr6oxfFUk-9#HY z$6>%btry@`NEMy@0{k1fdeX2CmjS#J|GH?gw~!iy=R5ZzrL*|yL$1ynNbO5P$0&Y! zAf^dFUAuK~aCUVaF$~V={sK2Vw(bc~qeXP@k7(8$-9N|RYf;^gFdN2qUx*H9p6=ba zb;|C3KYq1Z{SY{9`fq3*p??6&9r`g)qO^oXK)Q6ZU5U(W1B0*a1N4B1W* zq$CQ8vZI@~3@vmg`mO0#CcL&yb?0w^=_Edo%EMb+gY!Nl*)GfT`u&(O*?M>A z%Fz>G?F9Wz@K4k~#&51(2Fa83?_u*~y#%ReihdNWy!vObBu`%h%2fRhSbmWnf<5_q zA4JSF{ax5UUB4NlHbWl`c&5G*wq2}m$L}S2AuODwH^Tq3^-<`{9K8#|ZLZ!Gk}uUC z$L~CS67*iCcSY|m*T01P`TF(n?G<`G{J%g?1ph*PCMb*a`{CyT{Q&5NdMbVw>)H4% z(r4gziJlIP#ro&4q(tY9$5MSZMrxV<9BNDTwUB?Mei}VpuFnT&nLZfw75W$GZ@K;^ z_$&0!;j2o0C2YP*uf(WT=>yQ~YP||Q_2~=2U!y+>@2%8#VK%JN$HAYg^&0%H(Sxva zt-b~xxLU7;=daPXVA5TyZ-(CM^kVQ|uRjb=*6O{$vreA~Z``0yL2aELgqC{!Zs={$ z1F+{teK|aHll~v{dcFQ<$nop_@O!gf0Sg0q68gA7=OKDf?*yym$+Ad-NM1`Ch#k-ndWi34h+NkA>a`^!FhDLH#6J zJ*2z9|FHf7di_s*J$(3x{v+D{OTQJqeN^uQ`H$)Sq5X0FKFI&KUWDH*`kV0HfAp#F z>J$1da6YNO3@uOT3*on?^+eFO>iscZ|J6Id&TV>Ec3h&>hu$Btu~YvVe%Pf~fOEH=2+Q~A)$r|Jo!@TWr}NuB zNA(?$c1&LcImh*1VB5F)3`qD+zaL|ILSKp=eXrjR&Xf9g@WT&!ENu8ue;>a;>GPoX zl>QTRoz|a1PtRzb?*plOBGYB(fzaE1xs;<9qb%g-Yi|(A1+h*J;i9036ZdJg&b{T3jXk+SoJsBQkxBe)mi%6iocmR?qYs z07p6JcLAKW32?l#5kBsDK@f0)^E|ZenOz3Bw{tuB#ct2z90zapoJh+Oollcz5@il? z=Cke8`v9jnAH#3Y=}s7t;oL@eW+%Ye&esqHJ?EbU>~;1;6!g4;@H}Tec@_{B{}~dk z=an>hwbp@qxnlsux*nm$-G!-@(kpcueD7Qjg}u_~49)ph(0h#{9OZljgV!sahQ~XP zV`O@bUIn0qlq{qIM#>ih6x)0;4+5xNb2aOnJ~f0W)Gt>X3D5d$8568e&@ zXCh!~yYqPtHVMDLT5J+N#gJo@(1@tPCLs@g!6xBhSgFEVD@M|KSpYR5UuCCRHW5vJ&&-2pq^J^{@{w}c?D7luIyg3 z5tg;MR%>cM(dX)y*{N~ptxcbYd=a7Z6;_AtKu4l9$0k_Nw|5E}$Nd!p7w`Eur>wI8 z1kZoiRCCS+-Sb2<;0P`DT@+%I=Z1rXbK|tQmtbp#=j#s-`9-EC@ zt0-3y#@1uEkt^Qy3_9a+h)rvP>shinMH`PR89wuL5H&+wC(+U)_uVO&oTuRN-us~~ z-V<+ok?K0Zeotq?t-1K03-;(@^Bd)Q4g=yz5J;@+B@CXYr)Uzdbr=d8Uq|cnE&!nQ zkvl-oR9gr%{Rn(%)6>y90vg%h7}Ddvgxg|&1k^E#zcFV?jKl1Tjm8LdjN)%h7yQIu zi7|}rBZ&Ns;VnY^hp;*}O%VATlSN_~Iv(p4ME=HHLgGFc5W7GS`5Uv8#D8G`W2*#_ zzcJU47(qq6$?O=#-mH@XBTqwX-2$l-)FhL){z8UD^jkbO%?*nnxVcPdgBd9rJFkkzPCY<98 zke*5S2IrqJ{Qbre-sO1|Q>5PogdH}|-N?lKCK8!slX3kuV3joDet?M8{S)djAkK>s zz5RQR#grl5qgV28f9|p%ed%~gp6A33cS0ar24ry2$Mr7SUVD`XLeVuI6xyijqt>q!!)nWxM4!p| z4iws^1)wR~UTG5Nz-rs{8bK^m#NPjg%mmw|5wnn-`{R`$%{DJWti3L-x3a=y6vHs z?XJ<2aa-X;lqe3i7`R;Gyy@bnAqFhMkL`@(YZUg4jcW!KEz@j23xOA-vx7^PXtrMj zI;g6+TC@Es;DHFZA&(*X*nV?xLE?YCbrAj|y_)TJfnSV-KA@VK&IQNT~l}T#= z3?fhtU@Czk04o3_?^JE~7U0ItHYeg5{LYS8iQkKD*&}S1@H8dmHO2HUn6%VkXd8oR z%*(@-EjRKBs7rAx+Tj8q^9PA~?eY02Q~RSyuBx8NCetFD*(A@}q*yf(?aLK%7VT38 zjF!TPXsLx$aJQiv@;?n=yCU*$>FCqVpf|E}*8$i9*Ctf~n9&9Ir({dJ1K6_|r?h(k zbilHj_85R10HdA(z<5mC17IFzZQ7dv)?w$7_AvlvgtQQVG!JfVe*#dSBrO{KJCM|# z-WH2aaMI3qOoNc5^Fg0F0QyNBGX_}dL<=jQ$J=rp4zkV${VlSRIHrKC^t0K|Eo2?< z=m}#}HdvjuZ@4r>`ne44cmTocFq>pK{()Ub(zT#hB?7n|z{LyD`l+yeAKUMbHg#qSotwWh7kLL5~*q^4zOAU8rXriaCd3)Z-Xzju?%*zJ2Tno3mhpCO4ea&*=|L4DMSQ1|1(v*1l#)x5jA_1HX^vj z(E}!fnkt&j>kyv3GTi$!N<{14pOG?zAQ5G+D;qY`+C`4q1Q_|Y)yUdp!K`NU1Jg9^ zkg7hwCIyau@Zw0qX-}Z?4j3^0c>tA80tor1nvef4?uE1#XiGg(|1xVC0GBkiI%x~kTi(qzamu4K^Ia?JDG4(#wfW&w9Ic+PS^mfGt!Gl zzb*3=6Y*Sj=(Pfim?S6<;)X1gw>`nR< zb@PyVlHLaJAb{k*D4YIG>O&tWlHC2{4J$_syL4o2r?`oV>scu-+FnrU z5zWkSllknDX#2oR1s%SbC-tfMw||VJ50r&wJ7~WUiAF^9Q&1tZ#&N~fIxHVv5SB@g zsQK|zrleNV^0(|8O=*W|IX4cmqhR@YViP||O`W5eHN_^G+g(-0a1PO`rDe*t& zl-M$L=1Chv8t zI~KoCHPS>}S0Jc(b(e)^d&<7u15u3YN7yUI^&9L}gW|Ntb+rA>0@<)7XoQ9Gm2=!1 znqoh*$HH2IGu7HC%3qbNM=ags}Oe8uL z9ns4Be9&*(YX7$>kpB%rA@E)bOjTU1O|%Gi#Ho7g=&1?h29i1B*c`7Py;1u%&`?UZ>i3R8ngdmGRb>Z zbDk8?CCeSJL3K*5nmaTs?kCsk?^ZH6jCiD;=y^YA1~@ou4-6^=3J zZPF{Cm%-slF9LWK?USAaFdSV;dI-Q`gi6v~0RBaw5x{5YOHw_6$Fl&e2H?673mavo zy$>um8^9L;HWT;`K$i(v%6P-PWC2tI#MIPrhR2G^o$$ll8i6nCX!sLAWQgOAO9hKoUn7L3fe2=)RM|%;zn^rls z{`^=)<(>REYZWRGa*IX z{x+R6M{NolIM1{lJ8MXIxJQWtzE@@(4t7*D&&uqB9*^1 z+a!A?V9~*Ug*#ZSxLOb4OD!D~+t(QlSBv}pS^$`su8D^|KM^N^+8cjs#N@mwJnto*N~@uG^Jm?M8s z5e%ycsi+hxE>eDGyGzd1?&*0_u$Eb1VZHWTtS#15E7p++#Cxr*(e|yT9hu^VK?o-k z|3f}ulcViTCR6gY%FmWN8nF&-M%LrCg;CwzJ_vJI47&Yd0F3Gp0Fqx;T&)M*+cI}E z;8k9<;UdVe*f83(A@dRo6K}XJOc#Wi_E?y1Z;$B{#YAs6S(!w%;4EuOSl($1>(O(u zP7kx*1LyI|h1BR_7W*Jo%ban(tYsIIt+mni`C$F>MZ+}!_hb1 zdR@X!^49A*cJd3wrEa|(8)d2$H~Ef@jm*2a?a~%Wz2dfurovv|-IaT5qH48|6Ld>SNzNSBkVRP{77B(4I1Mh)wI>1x7eF zuIJ)3yaa>xf)d`*G^5x)HVJ)Qhq_U-0jvaYDS+gc6z4#>zQ}PCr+Pw=D@{s>GX*WR zUuz0lYYMuXg1Rft7D37-H;mE_F~fE(=jU=Uey6c|L7EC7%^Km~N`EZN0O{Bot; zqtjh5F?eu{)T2NvtM8;7zDXOK{=yL#PlVqFStpU z16X1I8`(L4_miCicmP0hnc_5KS|ousF?}MO5k!DprJBGDDf}w?m!>U0n6`9Bo8)U0 zr`cH{oLen9F8>%O1zTJYXQ&e$S z)a~P?qV`0ke5%@C$RU1FWxtDvylO!*-?1RiK#fdhTINR<`kRW*jV1r9SG*`G^9KuZ zMly+vsipiIUgUKt6`BcK3>8ymj(H=6rk8DRp`H40AtMTAU0vIs2VCg{J&IMspf#_&9xRJ5(@-- z6Xns8`;8HXX%rk@EZXl0?w=yP*pkO<+Sf>HW1K)zFP)BCOXPB1wWzTbIqpE^NWy8$ zP`MGIk>&%ik3cPe&PW|;8vtAmKy244RQzqBfUj(y!$uLITfra_(dPh??>uLeS`NOV z?cJb5UYUk6stDuBe2osa)Qfwp@Y!$XZ;H1Zyn`gKI7lCXjLeI7G*bl~YiVDlgKKHJ zmU)5VVcW%)wihY-8qn9a&{IlOFT^;T9hc_?WBEkxQq}G-eNil*#zkIc&Q$ZE@vjB! zk$Kd#17As3#vs~sZ|QNqEgVZpB`uGWG#AlJxicpZjdwAx+tP(<6c zjyoypDby{1sHA@as3q`s0QV8N6~I0K$um^jdiaC4YMeF~^PGnBg|lX!+&?WFAcc$2 z@1mARJ|Nu*O&COVs+&1XRq@p4Gg%dF{}Ky8m13{>Jo9Ud9 zXW$W5xB=tqT8MM04d+i6<1dRv?QAZMnSB+VYP!R>N=-_MvXY~Vk@!1u)HILOTmWC& zZi&1-8@<{GdK9Ky(z5{42>ctsQk-9oB-@y`fsOeTKdE0K`b`P^`ybnTw(Lx>5l;IF zl~*qWEB_cets{O;V<$AKI{+R_W%I$1*?0>Zm;J;z;MsXn0sisqyaYh0J+PoHtn?u&1cn?!x#gq)l;b{H20018Y{TD#;-HJ(#RYz&j``Sz{cWM znV(=;$(PIN^$W10~4=Tuo^n!>o>PdDIKVBAl@gN#e-u?dzg zc1(8xORrZ}!2eWyKFl2;%(+Iz;hNb}Bd+l-to!41VuIwF#NFraxe#;bWccCaf%tP3 z3H=GEX#JT+B-76TI=ddgWB_RmFm)CHp0fD>eg@3B7Xe7lFnp&bV^FW9n?mME=;S{4 zD8!hY4*YG608R#Qd=&htcKX>UU4_#0^8g$mU@m~Go6ylh0DQ|tHGuIeQB?=v83I}W zoZk$k(?)IJ8K$?*)Y~_kFF+rZ&;ADWoXR>8Z$8x|Wj$QXLTelNI% zv(eWY6vl!@)(67k0VwC|=!I;=M_vfw+BWO7F@_5fJ>Q=wcA!uyM6^V|Z-`QG8%@gC zF!?GkGPJH_w2D&cW(?HGO%*xhz-{N5ex{1h?u)cWvlHJ2VNE>ziV%IgBIv+t#DTGT znWd~YHt%G53;-{Bop>@-`t>K3^Hbo=PW>*abea(L=YEXTCiC~{%+aAEhOAGdKz{Zi z*g%yr88Fh|HSSYH?Yy?mmjWd&r@^hr1{54G5j<&X2IKhxvy*jKpyFcFAc<7Xo#^)4 z$$DJK$RgL;WLEO@>j3SeHf=k+gf&*+PUgeGCKpMQ zPr1nwOy&n=mETECzVE()CesL>z6zBmtb)Fu6pHU(f;~E_*?`}urXf+GGHLF@aig8g z$>$mj4JTuP6sV0&4_!fAa_ouIO_ROU_fOpEbHuCj(JlMl7XW0G3RQpK>+_W_*ONJS zxPW-$C1b`_YImJ-y$#IFTv;qs{W%75NfPeMFcrNfm=DK&$bDW}J>|1{La6$)ilrf$ zzeqRj*`==(xhPHeJD5Q=CQZ0sDkx2$OIqaXyltwwk)szoR@T;Ay-->A{f5>CJ z-fMwnB{i}if-RJ!-dHmD<*rEnuep#e1ef^&N!KX^KS$>;`iCeTLTN^*P!{7s0M1+u%&CmQ#mfl{?7VFHGyuP# zsyU3=)tfo~EF&k4C@WhE`yg8%`Xo0qS8~<@Ce36#jTDYvZyNCjdT082HIw$BxWNGu z{nq5SNp#Am7PSJPe5wyX#;H>boBlS6pR^4a-M|@!Hr;$U?(pP>c~Ex_m}wn4S(ky{=1P^a!b0Sk6suQpNeLjDP(~QG3Q*2z7>Clp?BPo$$!zlweD1{ zJDBuq?RyXPotv=lRn>+beGl{+r2uN(jHn3MTyV_#NJ2=ec5^}UkX%vf2k3@lmLYW! ze(B|(VxF5^rljd@8s1J4Dqr`~(RjfhXPDs5>Ez3`b+NWK9Zj`6TAchdC0(PW8D(}y zJmkGFjd}6#s_b4*OnzTkeO+0ZUh9N(f(qD?oSTR1>EvFip2p`LCWvoH*trtH*~gDG zY$(`X3^wHKoxryU{$KEzI|a|%Jp@M{z$V{-xNx9fbU+40fn&@l!!)@b*C(VghZclu z%rB{>2>Obt4Mxz5rTRoPe%DmH3!SV!L5~DFEuRG)=-sc5lQr~o(Wn34+xco^Wjl9r#)FqzdLfmKzluI^HMbp2iD}3{Gsu*Q`9UV&SW#vcp~>dRj1cH>v1(Z!ro3 z{ku*{S!4nJ3LjOm)lWCyKcu^{xO@~(UmS?m@A5a~+`lhSF93pu-bo(*Ziq+m!;=`M zLlt(|zi}JmD1bq3v}WLfe+~xu-$G=Qf+Vu$dkmXFAKWd_H+*;`20 z94EFk;(KPEM+r~idc;3H)1aSSgg zq;HP86TB;7vceg#vpdmra0mI0fzjWB!g3n@T5g1Iu(_di=rr%wWbfBmRF`gtDBv*x zLFRahI0*c8Alx?${=Z1XH~8z11-0bL|IK(O?*R&QPZ@4>%SJx5i6YsGhyGNEhMv#g zr4&3puE70?u@xNdX=9z;#ON8OCObUKdUc9_N>OGPlkA)WPIwVRdb!)^u82kG5cTMA zFBbYm-03c6GqN3J?%33?L^t{5UkH1-l1{6K%WbMH}{5#)=vz?>dSqcbXy6 zO$Y_-R0d(36EJ+sIN_cCz`o2)CStH(VQ4{Nh#dSOqd6z;4ZhA-XGRuJZp#O;m3 zP)SBP;W#E_A4(xdyN3H)P}oPY>;@ie;XBAU+#4B&%=_RPZkypvceRjNQ1s2x#)SFg zY;7S56$R{U2Bt9Z1taoA)?b&1Tzoc(T*_6ii^8U|+JN3hUuPm~pdS4^M%urm5$lU7 zJ70kIKL}|529Qg-lo7&Kk6x;M-j;@tNI50F{YNjHW?;h~^gsEqZiUD)A`!7b$aC$ffW86ma`t#*)kz zFz6?CtwyIK46l-hXcbQlxpc+Pv`FO^6hw|=$jMtrYsDY=EBo%9peuX<>Tq8-@F5DN zZ*XPdRXmZfz{$wolD$<`2ExZTm06;VgFs3Sf)jH{&bG(O0;% zO1K?3o?x3?!u35UPr8vf*0VrALD_uQx4|V|xTJLSa`P?dM(%Pb-0Ga+OHeb<^a)?# zm%HB0FTy$x1BiAp5jF? zFuN;0j_MQAogj6ua=)cyA1PVJEkY64S#is|1%H5AUhaOZEys4DC)yL20?2sAZq2(5 z`rZn?=mH};GMnt@g)|*ncCT?upe&ZS1ANT(=TJgsulHj{x7hT8W!B4L%04Km1-#7M zrW89o#mV}o)Kc2M$**;?CP^1`(0Y&HS-m7@=2oeFWd#bV0M5xj3DzX#LIAwDX0#Mc zwod6Oom`btwiKUT=eUI~brafB##`41MF<2Iez^QZoxouL4^TgXD&#%F}1oJBl70X9)}A&-XXWj$?TE(n6ovYDL;_= z=uw#Jq6QN4UC8YytI;7Nrn`+UZIxqjdG{zVnV3Ws5cxd{a?L1IvLF7P#(fXfu^S$s z;y$WnFMOYhgVvyycTESWpF9-L*h>NN5Y?u7Z&jHNcU0bsuH6raXeYIy)$|{<5bi^% zl2@q@X~A03haxXj9}(@}QRE2p#$3aQ{qI~Bs?~Nj=|57CjK1w_yO>?r@(BX%B1eka)}3be;s-?@v17WYmxkmQqfLLPj*j& zr^tcN`4Y^`fe$X6ib1~&FQ^3>dqLnrMC`1ACrGTO+{>8*zu*I$SASPU_%V@r&`o`z*&JkJ~%!I`ALUVZ2zAYt(nBCTpeC5HGqzR%Fe*I@m&;fm%3K*HZp+8~M3Q zclT8DG`o#v&SFloxT=OH$5xsgwKQw)pjlJH>HM8UT9ZOt?iwEgNYmc^@=P$S;raFz z;$6eJ-Zr9LPzwf+P-=tTQ-*?=pXBS(RgA^F7rU9rj35jy=DBV_J~g?82%IIa&{DSE zrTZ%ubF#Odx>``nv0x*?=;1XYYM_Ek4^*fr!z6Q~EHt-iqAJs5C4n`oC@F5HoxOAM zpD-Y}^u?q;x6G~DN_F$7vAYP&YTJA9;0=P1w9?bomi7%rgdX!%49=sRWAV3#P?`8H zej4Xa;V~5Isw0tBfOO8V_*S$s1kSRJCt(Tyh1&xOsf9=PeiRS~APG@$4zvuIjyarV z_alC%>~XJxh)hj4b`6ej)E3b=yWnNtb5U(#2K)jv;FpY?3c0%5nSJ0oU z@rQ4oOwTuRQN8|{tdM>ReI@-9tftRH)d}nHXxt4_KVItph}xN5 z_)|}=WY@a$c^K2s(hTEk35T&B^FUsi%|gLW(JqB2*z|OY^RQsPFZAxVtr2KknYBdBX1l_#;$-a=(SB8r``CcN`0)gv2JMSdHlS#i z0T=ML%%^WUd0WcpPGN6L4L)kTEp*J%NK5YTlOV|mf}&Sq@uK(e@l7T8p!d-GLXD@P z&s;>>q;!I!Ta$4g0N$ahrX7gr^d9>~H~PUadWbFH`hI1T~I9_rviX=MWTK5BBsP`$p3sbWHET{|rTc1)lUC{9MyT zsHOLSXs8ik0}nI?`Vc`YGqLJlo`$u2n@ixzdvUmz39czyT>=}g1H6v=BB(J3r1Y4> z@nZ>UT7EnxG2TNzaGQQS3V{~yp<~?WwOESjea1!6#TX8~$KKdT>*#5Ck8=oWq-Fg8 z-s2pC8fp1{4)1XeLD5G+LhlKKn{ps7y$75YYC3-e+yL(Z(?d;HL120heJ~XL`suj0 z5ATt4qu&EJdXLPH(hrf)dt_Dg6A;pSLR!<~;7{+N-wdt%D{53#Z_q6eTy(cw9SAiG3&#a85 z!l?8<{f1}{G)(WaMn*TIExpeg9lZb_^geBM(*#t~d%*Zm6FmZs-a}hM&RnQH$yp2k zlArhZ2#6F2WdAP6ccJJ1M?dkAeDyr|EdRtu@-^86(ocLOU&|o<#7FXVOzR}Q#1OBkK~&eq~GaCzKy`-F#kkH@;!{FpXf;b27~mo8_E22>Xh(@ zARO=x{)EiWZZLv>RU@UWoml@Ab{(*fIvlt@oJ@}rPo2@fC;={wcT26!9+^7|R# z@Ni)G_cI0#pi1-m8L8nlfKzWh9e+}vq}HkQ`x&WeuLHDyKg0Z9M%q78;9{xtdl_jT zF6Jimdl_jTlgQ!JpTR0=M+iPK_0O?edBi~?gXS8VY2GSV(|3D3Wmkv7-Bqlixq!ztlUz%NYk zX8`@--^)1uT9Sc(FXQ;@82kdF!tvKLNWYgc=u{Fc?OULF2k)Uy_;qS<6P$3+X@ue5 zs5oIvXaI;)!sI6W{Ph#ZF-Xr}KXDqd;ODQOIFmto{`!g8KI0-Z5H7GJ6}~ z6I1EQ=|kph0X#gFo}4~pF3s1Gsr2OZAvu==PEVyLrw_>`*~X>PlhcRf6VFMh#ng5I zIbmk%L-;$S`wx9D;PYnTHN^nhTG0`j|cqMEN_)-#i zjl;9J5FigIoJ zBHJv*hGC~wna2CD{5*%&m>oFp8cv5|C)b&(;kco$7pU^kmIi0oBx&$A8vMy`;EY%* z_z!@0PNFp#85HR#U8H=>aD$Yz^qj)ae_iZGL5Le(*Pe=}!PdA`b1P&%jeb^Ty)?wi z#4mTLdkwoEf(N6}(}W3+-v3-D_x#ZYb3sM&G5T*F@C5Ew$)k*q#91!FWgx;79yM z@T{PeFh$@2G~ScQQ0@?pe(4t(o^+=Uz1(fj!W&NJr!Wt3iEuuKdGao@uqpo>WuwcJ zT%yG{Dk7)z^}3Up~?*m%VFIZfu214C8Vq%BMd`{DyP+n^c}iw7GEXzeChh`9j~>On5yiM!hQXPJ<47ebWUdC;g#vGlZGu!`Xo4@${F8@q9NM zxF4ha8FZ)3cZ6^!I&l}cGeI}T1w4j-cWo?@JSQaRgh0;hDL+Dl!{aq|YEhu;E!H+A(gfad{>$`z``%*)6- z(EIfY(*w=AE5Mf?&Vp${YdGS@juLk|Fccno0 zN6r?aP7s|UGF}b*2vzK6YOzpis0IBni?N3@>^i}d7}pBE7xcFV=ywF@_XhCakKsq0 z7M!-cyu6Cqag>)GJ;6DN(tS78gyS+}ku39p$-4-PhMBynJel%tK%C+eZBu>;3R9?Zz;UarIiD(g?`AWK~6e}L&k3@VpQHp zw~o_e#ku$>if8eW^nX$FAz#GBIKC+Qdn)3QS!>U}jb`Z|=3_0wW3twsO%H2z|0u zsJ}|`{r-$fI2t@7q-DepmnV-GhNs;u3=08_+Mkf&7cGV?;HN@LQP$m`H%Rzhs{FPR zztc+`e)ShH)8aR5tvKClP{LAhlDRVM2EqR~@O?g>!heo4><+0ULvJ&cZYCL;@*GYJ zM%*hTe*?*|pGx7MDEvIgjNIC|#<0Hzc>Y~@_II9K88%0(tAho#7@2hhJx-clOs2Z$ znva4tszk1tRASn2CNZUg<|98DEK?%MgQgk*=&lnr6g*F%;0HIO_%o)b;-$Bs_;)0v z?h8~5Z$^>tUV4E-!4FV0f#M}9=4=s%dr*2gk(RYzo`D`RX;n-0^qU?#cyYTM^rLlfiMef$+(nIH3~ink}eiJt%+nnXX+hNOG8~}YGa}DB3h9T zx#iA?dxebktV1AeIulPLh0r4*oTrNL2d}k$EdAglRp1s<(Nh?d(HIv;e@QjQ&DWu5 zGT5mV8^kE6DTcx;e8i&@WS1ceI5Lw%CbaG&bM%MdGWWC=+ z);|i-PSCgye0i|Us(;=pvvmAzQ2(bw)L;D-E0{erXr?{S()Sh`>_@H#OwR>i;Jwgq+6PB$3g!1$oiyEaAbW(3LIH=Tx6PI`$Z^= z>{kQ5yM0tGMRXk{b003Fp@@3Fh&k06l`UxM6q5f(9a3DWB}22^tgNRYG2?5{+hK6h*nEQEDK^t@LmIrq=8;lKu5!Mqv}5yGLc+1RPzn^A zn}f_q=@OgU0z7XNp8aF8eiN_MZK z+5dVe{@^9f1#N%pWDPCXQE1r&RZrsk+_`w%R?XV3r`p?4G{4Tm#gTWQ$oC#_@#`oK zJ1RX;_*?_tqW=l!NaUmbfQtpkN)sFrBkAs`a>QYL)F2@8T@Y0ywVusx^aCPsH_2rc z$#u@1Dm52xMA77$)l~aes^z;SxL9XEcR-zeB7^UP*15pk zTf=0uhnivTt)V)Atxo6&b(oF*0iHB~TA15sSPnXoGIbHE%jVl+BGkt$=K^#44ATud zO*xMy&W3KZp_@%{bhG5~r5HNB+--CUJMtq`?SmK238AA4kH;Tu!`ICvZSI#ASk?ID zX5EHPMwX*$(A@wE089li>aY-=02RSu=oE}jbV)9}5cqMNg%c`z7!18mG#`8&bIu4b znKKT5oE(z!VLe#!L{L;tjhez<{infWe4w3k~`Du%`mtZZ{2h zhhIsDf+KzrAjN=(JthjFL#ii*x)XzF3($@DlN8wD^UsH0(IxofH@gHD=!Mpy>)k@V z`Z~Y=wfXs^T)Mn|JYQZf(96j~i`*i;Abu7Cf!=5^^owqhUN9c~=K{U#eZn_G1>4aD zy-)aYsNg+(())z$P=VelaB?(MbT4r99{fzvetgh-@CyaE;Dg>Ld=x6uYs$kKiu79f z@Gs@-Ed`#rrJz9XjyU1IPysDq^gdyXTW}e;(EEgELXJDw9Y1Jjnm6dQK~=*ByC-6R zp>a-H+8m%TO9Tu>SB)NS`J32JSz=; zFk}F6JUiBgIzMNWJC(bvNvq|7O-^>EGV_eX;6u)%dg!?(;sgG`_ugPuxl z%iXGB+ICtR3b=u6hOn5y02-Z}3JeKy9syXvr~>O1^@UYQdm{NR2r|VI5R?zgMH;%S z8b*Y(&6p5Q4Gw{y03GMt03UYaoB$_A;A^)SDWuEI5LHbakq)28%Yeg@lF1tBOp4zb zR=p!!#cYDF#>KQw*C&Mx@kZ5;+nLia@P)X*521^Ly!%;tQLGCxZApyKh)~M24Zc;z z09^JWAxVq#nsVWVX+gXy#N<+}6p^J^6=Y-j!kxT2NpOB7!v#66y2je#=IIz;+{+zf zW0+hlT7`I|)uq0dv%KkPGuXkG`RL@3>T>TKR#LE5%YSbJc5~WTNH=!b| zH!L{YhX*3d2xBg?)v19daNyGdrsE)m_-=yKC}oBaEv_lOj;2^dQx4v!G2x6rOkh4^ zOaK}i(QRr#YSPCv=^>4Jm>wL2nYS}y;v#UzBtY<21Wf*5cVOY3L)n$9P+sC$H-}1ylBSbe@4cX z?ylw*XG(Wx$CT#Q`j+m7#wlygKQD6LjL9v{t=(%TH??+8xum%ZUo!kmlFq*V6o$N| z`I0G3_4V-_XKH7k!-PYaOm4U~mI8J9*Lv!1-jFlLo z)7{#PqJg9Jtz9jG@9gSmZf%N0+d8_Mqs@&%(b-kk1zPOv&%V`qC~y&*wY&25HhK~ZtOw4-4F zBnkRDBiJgpU@V5p9~LU>Ne)9--_{y!ty@hcjJ>I`3y%+o)VFo?M>MVxE@0WJJs7c? z+wkZav?j5QF2JX~m(+uvu(fZ5l8s2sO|5k;f@j?uy+K!X4GkTrKue=J19x|IifF8_ zx~^!a5V0u?r&*VDH@9?6Zf>0ecVrQm3N6^a5pqpXVi)eSMqO`RGf72!>sP8Rqn+K& zBsZ`~@a>3wT5E?1kkUxBkKF%c#wsy*K-XP2nlovs*%AhI#|{Ul5*M z@^mzvY^5pxI?L)a-R`wlhS6z0P$HhIV`3zcPC9 zlIp5|EUC?&GCe#rd&;Ws{?2O+bBnD!l)v(bx2_{04!YqCG{1S>0u(-S!v}@;W~MyA zgWk-PpSYX7gKN7Mh0h3&*|g}PmGix~lDd9)!ODvZFRZP7D0|@ycxF;4JeB?y(w__I zPnOZp!Ibc(o;gLe=<2%5_n@o8Nvw(ME~kDECuMkj4|?xkc5vYnXym5Uq+AEeQ)uvC zPYPwY`>uW{d-e{zrvQEHLEonQ$G>Ah<;f#e-ypG*Bcxz_m0sTuR`Cd}^APO&D zo(`JtCDnK@IpHC-*$YGM!;5B5M$JDjqaHRThYvm+!o6)S-1z`bTj=F|IMDTmg`-~c zEPP~a%Z*(j84o^6F{u?+*w^?+jl93BI$2B>3e3cGmB#p~3X+_MT1M zv&DO2-S(@!2iy&-!lcNg6zV1^cfw}+;S z*B$i+9c+NO;Q!aP+rjeQ(7{md(D1PHy~u*+YiEZh4K1pDs5bmWZBnv#+Z7Lj;$3eR z!z94FUX2eQ^lHLQ-UliBlTy4V){tL6J0LuZ3uL%^;GaoeQJ6Z(4tD|0Cl1-OR=wsu z6x#4@@1JY;)MnQuxf=i<<_UKlJJ?}%uhFhPw#=9wX z!yVr3ebmRdub_Fn&x5kwy6r=6N74pJ{`O^I()nv%c+VX#dZWT*mA|baC2TcAyXFd- zs_WrXTfyUnHK1_49}NhNqdEA~H8fR!G+==@sK!eRZ}!I2c*_rZV?sB3?k%X_!v^ic zJbK>?O>wFdlc7@mh8SQyyeYJzpW2141agZo6X_o@D1UOUPgF`_p5aef@=-9hG)Il z{e#yI-R-^a?Dm=vY_h#O+>Ga+-?;I4@AvNZaMPE*^Th4mz&(dHCJpfR_mQCxJ?Cn__j{e3Qhyc)qn7``z@Ro3P(S=!G^FD;NLf$96 zyV3oJ143inY!q)?Lv#OuFe2yXjT^l$Uba6x)SIw5Jmt1S*|T5X_|+#q@*Y{c*?T!O z3!fR@^zf^%c$>X&^iX&zTHkdAS^h5`$Jx8CfUA=H-gi@>MOr#u?k3|tk{YI-U+yMX zz9k$Ug2MZ~q=jdb;7Q@$@Dop90Nx&i>KY>KBN1QXIQm#Garj;eoFz1UXeeXw?CjcX zijdi}!}CDCGrN-+(KzMB(`BF3RzqmX!*-di8f23h8AwecE^CiUlhaTJZnJdDy zK-`%ex`=)3e=Z}w*Qaa`p9Bu?nQm@P*^a{BSf*Q3W~~nQc!liV#cvkBiJCVNa9nR^ z3Kk;oT|6os-96IF*6qWj#JV^U)dEHU-aG(nItg`%1IkSUSgb?50t_~(N;`>rBIDah zTryGU4**_E2~EJG0YlyceMIs~3ft)cmhHb_CvWP50ayZk*#$?r>Om2(i7CxSqm_x|CWC-#wdR#Y%F**S*jrl4<7^RC-}RsGCZU zuX#0uuMxZMmDpE=ocCyhmSDq1yP(hBjQuD3HE{cXZHpF^s$UKtw^2*5^}+^1w^#v? zvizEuqi8mbYah7+z%YuHijyk@c zD?Z|i9WD#{23PFhiZ{4oBP-GSgex|3#V1^`$=&abt?}AxLQ_ehf8kx*T}<#7Cin^n z9$L45cX&Fj^N4IilDrgen@4s$a7B1lu59U=tC!* z?#-aBM)z&u?54>-f?3box-mP?>zlP2k|d>s3Wgj^xp?98x8J?+J1knbD<%B9TCA1_ zd9~df!YVd!bEsr!_Ux7DW8Yd@lI{;hnHU>3{sbCMMIfqZ6S!|J?>s}{-QLB!!EEC? z%4CVzlEqWAK=3tAy8pl``Wma~PcCE~y21NG=v)kagV#OFOPfX8v@Ppsr}CZ)Q+su^ zjr#67B>1lPq_N-A-1U^i6qpdyC-qGCCiLex@#|aR(C@QIp zEXt{II_o0Qy4B4seT-d{Q&yIr7opRj3P$A4FDlMs!Y_0;H+DIAbZ~^uQ%q&?!boNA zLMKvG(bQ4bzLGk)q-9lS^JR?&#HmatQ&toeUR0G|mCu}_ATl+PPNI!;v9y)tMY)wZ zl|_s4xk=^x^8B2BnmC|2(1winNH+L!>iSrmE}1!X3-Bw z>sPlsi(uk{((+uR&zjP*{1U%YkhBsFooCU(S!B($b6JwY{K~@eNI5$ZOVmP3q_|mG zQBee{SeRcKkh-V@uGkH|`MjE2>pI$<_KvptXmg8kl?u3GUZg0`coozpzZ3lBQ|D9 z<5iVrp$)1G4T89Idu=uLf;|k1d;9BdZ%%*SGiaEHXVrNTID=QE*+Q^71Dsf=85> zUtF2P3G4C9*G74Rdt9e%X z>M+BjBBFwWGu(D&#g0g3h(KZWJ(0SW<~m~yQJ49s5b5xJpXLrcvX<;fAtAp!vKW&O zAH{G*ai~hvECL*9ZKrtx(nS|V{ElGSML8vTMDH}NX+{`pT)i4+&J;XapIp>^g7BqIM83MZu47fB z5p-T|B(G=@4|X-V0EMe)OI;J<^{VFf2x3+jMFs|!dMSAI6+1S#;zK2z#q>{rLfFjbwaWLrx_}opcee4YrnSgvZR~{};3N?w!u7ok;xh<=2#`h#JZZ2D zQqU_ZhI=r&q!^RBq>#OhhI(G4xa_HFL9(Ldq6%)< z-Ac)JV?#QoXou5McUd2$;}N)ZM{F&mfKXad6v-_{G=f|m^rUw%rw_PaRmr@flDwRG z7)!LirA=a#=%dK!!z?qW+Uya0|Jij=$Hd;;^k+CkxFE>tXVtq|n zp|Pf4oX#Wji_icDT2)+HVj?SiqOY^QuB9c?+0ur~R8Yjcwl+iqP?<*gCG#UiQ_shu z1s^8MMA}=r$t=+|vP^V#UqV5Ek`!*oj)WzGR^gQm1~)BIRhA!%H;r&_jIa#qtI5eN z=4NF%g~m4;+9K@1jO89HYAY)97nv(F?QMv124{?Pp+DT1mno=f2%QGpcJU^0GRmRh}v#qE0^xHQ()0Tp6B$I|bJc2Hy`^{#C0YUIV7IP^>6B z`eQP$+_EZvEi^ihRKpKLHfEHTzbUHGOZ&m?A1Join5{-U2r5kgCU1$ z>aBldGf}OJayf2Xp0il;rV1|00vCWI6*&d@k%Fp{T;wPvhH7zXd0s>_nDYEW1XZkF z?R7}CpcJy~{PNP0(y9u|Ynvr;%?aRPbnb#uq1*r z6qZz_=T4oPJ~LxRMrQit)m`1KjkB8?k!m#8qgaOpc{Xkhn0(%h^vO->lS`+iPmZ>> zO{T8vyC!os0U3aV>FZZbhIzXh>yc#=1;QG(Jkdsn0zwoC7_I6iPigKnkp?+aa|@;_ z{DwjsMvA4NGQXyh7ZEo3G)ys>Ed6Fd1*ahn0(@&DQlsA1PNxg$0`^TXSb1J)NpUSR z#4_ZsxbupzchHH<*=r}hIag%28B1eOFB!h7}SGIMxU`6fjYC{tfY)N&R^e?I^t}ODSD(gc+Y-xE#1Y;_3 zu+L#rlL^|*lGSPQBQt#~s*3Vr@g7#ss;R?%3tIFW+MHOornwyZB`N?BPBV}k3 z{Gr5pZZwV4kK+8i`~ohMdvaWhAf`v!qEWo@^l1i;5ZUeqEIY+ny3zB{#IZnV6yNZtffwKC;XO^4r53l9;g;BEb}>lDtjE`d(k( z*x4ybdMOz`V&*hv9PhK6ThDW#5Sqn2)yxu7k-I2TUQ{@raz`xC$Q&?TD2Xt;lU&$6 zvN)d!d_gqhT*T7T{)1M@h%8#1-nQ2@WCDYaspf-%0ZP-&#|-mfHvZNUPdQ~L`7KOI z%NRa*i)+$wQf(mNjwwxZv8sysNL5s4MP=3Jv6S;mah4OK1{>^t5H*n*7-HYVtTjIv zKy?xcD39dnJHBWqnHIr zSzbO=QI%U+S{|YOE%JRz1xSA5%&hU^QghY>r}krwxB^pQW1qu^7;>P*5^&;JUWN1u zMJy`8O%O08fq;(lD-&9M3-V$>uEHl%#DPu`)VA|-%FByzl*Oz$OYtL8M_XGL<*!`D z3lzMRk0|}D0#lGSMwGug6EsXfa?Y-fKC>LMZ=ug-Wa=~~F`mCUjD#PH8eC1SHu1C|FJ z&RMEyr%c?DLg4(0bVhtBKkB6Lc1cSe`5!G@#D_-G+PykaMv7Isqbs)E%&AVez*WpQ z(Li&vveNRLJSAMc7up@>;Jk`kg& z+V=H?a^W1{e!ZV$?_;h+~ZiohE3?Y|a`M z;nWOkxN(R4MUfI@J+V++3C?C$195S>VJ4Ft&f*Xp2acuAYN=_({f9==yxFJ2(Y$;d zm&|A8!Hf)NAiT@dqXm=Iv0C`dof}jx=@-U~xHt{YU07C%J+G0tEC;_B1l>jC07d3( zq#Q>QW>6IVa&xe2W!J!7pUjU%rJ);VvselE$boj{WD0U56Lt}-{Rvo6i$G9ncAkMP zC(Q%aJeD9j)rQ{6^K+R8W+9!wBE%SPshFQ%Y_{D6SS$+{;Y1Ub>d_p+CVi=MwI)G~HTG=n2ajIj^ZAb?^Mx zxT7nbW<2i^B1n$u)LHU!PV+FUijh#zStza9u?=tke2Cnsxe3W*3+1M~N2LR9%zugt z`IXq_@y?j$N>mQo`(^XKhP!$qt(}x5p&Qi3QBvUf0>#E5~nSGSg=n zZ8X)C1?B((!5vpDr~{)jiZ3mcPhdt;>Ww&Tj9ZC|D*nWR%a)Yp*T8G1tZZA|IHjYm zr@3`XUR!T#OIuw-Cok=jr)Qj>kvWByX@ndIOB*^$r*d)9j>Etr%HH7@G##+W@S$f* zT_=uxDMya*SsW)V&*u~Mb$ z5rBq3vJ|-wNm5i{wtm=naC_Q&!rROY?6>%0Zu0TB zDsXB#>JQ5)gwOq!=AP|Z&Xer@U?}DaAk{w z7vbP*@L_+=jnwJoI%C4({KA}E94*ie&@8ObkmLY7|Mkisu8&a0OMc56b}|*ODAZQI zKEQXsnAwP|r6QltJyzHC`TJsoA&NpKja^;0hE_|7esZtxEG9^go5fC zSGRZdQGkbv>H0Vwh2mO4XBXl}yHj@w4wn$^ko54nOu?XqF0t_iIy8sNP-7Yru9)&6 zO>1LgLuaJB9f?{a$4kB>r*~kd<6GnOa-Av#Q}IQ1gIccOR|J*h;IV!?v_y|65JCm3 zCm56=)Y2Nky3TS1e|k=kP)jcf7K933o)IL}(r*O|LIqEs8YI+Gy}m{k$NdmL>@ngbF@?evnX0p9mI&3O+R}NT{W{*#-+j1+Nv9(W9x3{Z-?Q^Nm-j zpjymzmMN&_B3_lg@jmdayi7qgmm$B)xPnTDX3O3 zgj!NVfTMzH1w*JMH3SG1R4W)lEvX?usGwTG5Nb&c0YU|XHmTeZvxb56H`}{>o0L5- zAU!67UanK7pxVZDD%3oM=}ZKwiciKEQ1uOz&YMvaeJTYXnPgC&v*o(pac7%?8ikQs z*fiO1thUhEgD%dTVrnbp6DS2Q%nWo9Kea0M$}Br?n%}`P0sk07Dfo$?ShmXPrit2| zkxl0sq}rE}T2Ko^zGX5O+x$6Mu~Wcb$R{_wV5Z^sefebM9w}Vl&(n&f0^Sxwspp)R zR+YozdO5cxk+D89gNtl(Nfabqn!9-~w6;~?G4Y{O@Vd_o037cex-?_!>S zUyh;FtzAFVRE!X?Duyx~Vt=zo8F;>Y!hx4d*r@ZvN7>)STMx=7gz7lUH0+j7G(1(j zYP5WM&cLH_lPFa>icODdogS(FVpAO-nX1*pHSoAjwR-ptKiX8sdr-BSSEB(Ms;)lr z)mJ@LbQIH7bxcbJI#WcyAf_t?H;b-9POqb|4h!agxJL^*Fd|$bVYk5#zhw$)SRgWu z3*Q&D5>Da2`Km@ATEHV~_(g;YPmN8;hkLZ36Osru`ra&yB;gd^D>xpB7G4ssB|?Qi zjCu0IJz7vD6QQkT!r8%>aQ5peQTdHA)hnou1mX7ej1oIcmrq8j?lt@}QUkCWgpsQ1 z2PKL!QoT&$C?nO!HRE99CW(=!Nx)-dg6OxeN3}gkq99Wy2#Nc8?D)r&8UZz~G187~ zj8x~pDi*W$^=Kdq1fG388hD^ul-Q^@lWf_&7g2Ewv5oI!o2OI+1`<)LD622&Vxma#T>*yKNS}v%nY04T3Nf) zuTz|Z8}R@S$Btf#SO3-(O7R-UE)ic>yn6LFB$z4Q_H4zgcdOefUR_pwQt|4S8p0HB z`=;X6KQ+E7-VSbxxBXP{wqq*Zc232sgC3R)U-6p%spBf%c3#EX4y<_Fi4|`Y{lwz?1(^(ZNDL$9SI0$I}_n-Zz7!SPJ~l`(wR>;8eCeU~Yyxfwdx+hflHM%vSWkvb1F z7iXky@?-LWr#(BEQm2P{JtNia)teb3nT_Bck`F2_jQ=NPF@_rfgWYGg`<-xeJFNkI*R+}3tZJC7k=@qsXwpKJa? zbm{?#a&~o4WLN4F!314lb&0_FkM7K=2lWTd(+Q`rP7^d$P~C>e{?i0q6CrAA`w`)6 zHzJ(vMTE1Rh;X(K5zclY!o`O>C|(kuSh)K?m!YetkSzcA9cAgi=O}TTdXiQhq6s`~)P8L?Qr*r)d z#a$=|tO#ds_h<;ZT9CB`+`_t$e#D!CLutj~(M! zemlr9GCs=1Cv{AyVeW5op@s%7!x?h zR=8MjK|Lu>56_s179I~UQ8=)EzwMu{*2xqKa#oz4z0rUb)+xum#m~7|irM9_?7fb{ zdUc2!$4|ugywhIVIW4r;c1CJ`_vTpRMvTCxGvGl!8{(DOW&Gr>IC(__ygrqm=tCln*K5SU$i{HqB%p}2Q@CUNrw zQ)o|5g6XoypDsELxVaVrb7D`AJtvq_7b2YqjMPG4R_y7qX9ZIlmyZ3lrv*-MityV- z{`UW6*$gRO7YW_PDBf1I;%y-mACtg9pC`xD|DhbO#ZJh0JSxYt5=&GL-6`lAu6SF% z+hoJ&ZyX)(dva3h?-0HxG9m!_bvFNC-RJc|Ifh0k3x z>PLFw3fjXLAERr%6Dc&>5MYf^P!AC3&KLBDX9BDG_)f=P68RA|`qyUi3)3P?2~EBJmPL;w6Y2OM+{CeJ)oCru)iUK2N}y zK9|P}wOn&cUGGB9R6nQJ=!q*uwg_jW8%#h6|K9LEpHX!@qw07)#a)g0M9#%@AaKac zrKYzXcBg8>OkaV=w6c;U;es?Czhu4Q8O6zUqx5k`x@nu>LeiZ=a*r=g+2aDf;iHx- z_?C~VR&bEdyV7(L`#Vyq(tVPPVwDDpCpjjUGkh*_;|zn#winljbir1`AH&=?c6GO$@X`tdY+0&Fx@%DyD z@p^u#dmY8w>6_whZK__b68$7OTQ>LY(9%}HVlh!r6^Lx~^#mIZAZ^t#rL8wc+Dc=j ztuaR0>SCm=D@NLiVx+AlM%pT3VZD}#nfGvy7W9e{5!(BA!s$+3_uquGjYv41DY~a7oE})c6gXB=Sg%L2Fj~;F zCTeV(lyLUenQ)2Bt684zk%`WJsd+W{er_nfL!$F%1}-H^)Z;INfu1kXQf^BXPZKZs zyTa$_X9<*mXityb+h{p$*3)N9oAHYcQJiE6X#Epvo39;y!S{YiN8|Vo7UvihJFL_V z$}wq23T=YS<6f!yZj7Cu_t;`8?Zd}rce)(-X}H8lPL!QPoOBu=ua=0M&{1rnV5G9; zvjZq-_Z>@(>*WyS7WoA2G1aKFHw$e7HIA1XZKulTXR%Y-OrcG{PUCrD%qnY+H4>fp zOJiruWyhj@UuZsw(N6k2+9N`nK+KoK+Ij&6%W(sxANKjJ;x)|c4GfCc6;h+M;_d4Z zinlLGDBcdpir1j5_dhD$KG;;e24Y=m6|c*yE~$#Q4>J{S$6v+U@mBG+LnqMh#mCSu z;}htY@d@;6=&cnIaxgxDei@%Yzl=|yU&bfUFXI#Fm+=Yo%lHKPWqbntGCqNR8UK0u zMFiTL=;FiP9$T|aX{(iyw$C!sRwyHFZ8FkUB_nM;GSXHeBW(>b(pDcMZQU`_RvaUb zrM0~>`wPevfT|KtWSomY9|7B7C0H+-XQwL7np@=_9MXIyq{=VCi zxK42zsj044>g;G8th4<-SZC`$SZA9dSZ7-!SZB|PV4ZE7V4ZEEV4Xctf_1jRf_3(^ z3D()>3)b0I4AvdXm}(b9V7@!twu$0xdnn$vh2m{HDBiY#;%)sa-qyY1ZM`er*16(s zeJkG9wc>3(E8fSLs>J4V`yW2CLMx8&xi|B+7|pDdTSx=KEQdQrZe^0Isab;eA?YMgum zW%JgcuS-S9Id6x;Itw7w5#LJN<K!K47Om`&F>To2{>To2{wfQ57R)-^r zR)-`0>M$$zUArSaIa1K=JdIPO%Opr8Sg3UdlN*^ z^}Puq==xqA?!6LaV3_s2T6iQDdKA=cAHKhyAkMDuO)xLl_uAUxPHcTKGC`bO-)pN; z!=66Vfo@DtSXGO|U3=XpGS%zyz%8o^UlNPZ3hDv|ZmRrGKklE%SGTRy; zQ(LS#*9?xx6uv@mm^KPNq5`!{6z8+4r5+6>q5+6>q5+6>q5+6>q5+6>q5+6>q z5+6>q5+C+gVqJCo?AF&s%J! z^$vR>57yaJI9O*}Bv@xh!eE^}3xaia7!20gQy^Gp`+cy^4t&8nJF+Fv)11%HlV6uW zPkvnjJ^6JB^yJqi(34-6Ku>;M0zLV43H0RGCD4;!XX{DRInCveWZ5DCWlNGE6}wTQ z!4~-pIqhMEW6Z0S`T73!g=z(Ll}vJaoOJ*D`?m-o@l*J2!JXh-iJ#*e7fzor|MX60 z%-FD2=t`PsdTx`P_67MQ8nx_xiA{u4IQEsQ_$z!*2))w6BioO4Md^4^%wRuk_PIMo zOJS*ex=yWvsyx?;sQPvMpi)pr3M$*Qc8AC?&yjBW{O&3gjCEI|pqgy})XWN2>P5+oM)7l@8Z?^HI!EJ&{aXOszsUzc(=vPIPe~e|D3jW1M#W~?X(Yt1!2|}up3>VT60;BzU zT!ed5KH*Lp3E~8`X`#XRz2I8O-G3yXVE9+{dsY1)346ghTQKFL8vty+6jaujy*9@L z6I!XC_cs-BkuIL3+7?GeK3PO}ow%Dwl_aqFw7J|Mr-_fsClv6ZD(CZP6KeZU!suQ3 zOmHSSOngXh?m?eJYfNWx7FOCZzTZ{ZV(EgRn@q6F6otmIdr;Us?6a#>P&ZGb9dEZN z+gm08_6WBnWu7_7bxs+2OrFXhfyED(;}|g`c^ivy}v~J7-2j~92a|kk6|D4s}H)_yIoETKa2K=BmyV< zPa7+nBjADe%e6;vS;e52xKq5iUbLXKnJhNQH2`8~wVz{R4Wz>s&z@(}03`qT)$ zeGpxLQJf==R&OgT^=~4LTXf<{;+)YYNp$m@O_RXweY3ghd9Q!> zd7M0YH|ONjjmb$KWj@+Z;bS*iDl4MX8z}X>Fwt0S{us*}BK!NX$9O5IJ%UPQ-|5R< zrl86IDqDmEs!|EY`y@DY?jY~j@R%7flZD7*Cxfp)>itZBCM>>^yu?3^-H&m z)Gt-&5E-fKyKedzsjI6VCmN(IMEdm}Mz#r3cl?ahJ8JcAT1M*geDyn)jMUp=_0Bj( z>QSBE|H{aa#BcqE2_yC3Nxv7#NMBh3sLoVDaJE(mXDfwp_RH1@yc|E+!mk~+Zi02Tj)HZzu7Y*8&id6^x(n9X zItA=^VF8cxzX@2T z;9wuMbgY0Ad{mjLt=dN|RSk{yx7K9}>JppaaN}x`^cG+EG6hw5XfKmxo_!3mB z1hMX_Z4;rPZ4IDQ@sIm6rc6OyU_oV@;)QgBiOIPvUe0AIr%oVjENuS*Dqi$u@uDxY zoeK@)N3+a!El}wIW22Ef>SJSiO-9orhTvJH%2nYRX^YQD740jsrZG|_nI^j+M#j1{ zklitPz1=V2>~0BX_ewasQ^MJ=k~0jZPChY<6#TP~TB8cXdJ)X)Hrwq;1*uRpsE|$u{~&@RDxJ{%j3ISt|LAw=gt_7f~xrh$+zl#f4b2HKoPsv&`nCP zE@mhLrHe^aFQC0aPjEhuHbHRxK-e7@TNlm{uq1|3Fe)fG(yksYs5M-tT$QuhUo6UO znG)z>&9M@;2@=oLe=*i$o&+iF)MK9nJt?d3Q?Z44fgH6i@{e5O1kROpR#!U+(RZwr zP-#p2wK*;&)P-bmEMfg(wNX-B!m7*36R~Xcm`4SF64U%8W;J~*{5j_t{)jTRHHxK5 zx7--q9mg7+(k}Md#SKm$;MpS&sD+>G>#_UP%7t>?!_K7lbUi)OkTddFE!V}Ke`}MB z7N+B3he>k<%#We$R(d9+KN?~-I`>I8568MwP`iPc2YR)jH#3;ldI+LtiUzBvHF`#7 zusVAE@~6T68&W=e*^XyKxg51ib^Bpm$*H-UrDgFa6H;XUzkp9U*NJ?6mYc z|LEAYo%@%VKI6JRQ)qHyw5zI*L90(((qE&1m}@RR2Ce=~#rWBV0JLStqE#CGK@-sK zJQl6e=x<(tcJHxhl}3N)0JO)P<2&aVIlm>JP^*Hk`lzJ}z9uO8&9kaFohq+K{f3na z{?bQPC>Z1w|FdZEye?8+zwvohD;VT;QJh?_OM`d(hE)olc%d0SmN5kf3o7v#mDeGk zSEYg{_`E6<4Dw1a#vIf6&*c-kSMYrwRiWVGd4}4^rKO7ni(5k!j0(zgIEfTmsi}U#z`?qjQ5D894TLr%3joFJYyEpZch31yxM*BY9fVo}l&r)!em*$x&7L zn@oU^gkb{Vkq~r9ARx$)kQfZ8Ox}q}peH>u1Wl%QRo#YKpQ&3@yV>*aZ4N1@sGcTOzHHRZz*(Mt!!FK@3m&kFI-&9M46j3@*YmMy zTq#t~aa|Y3+`n-iwP+zrB8tUs7Nx+GU}&P&YG_+~!O8=xUY+(MC`EFBwG=PN*wDg+FG-!^>#f z+0mC-QU`}R$}>@NB!9QzXE>*0?OeIxcvg`keXS(p`~q7|w^o5`6-cW)*=m}#>T#`l zq}A8i>T#BI5wWn{kXErI2#!v)zEhX5`r#OozQ4r|9$=}j<1b)3%#*Iqf6lS|Dvk;2 zR2x$oYP*F}P>~^hhXnQUn1vAH7Fe&u2SrcERvA`nF$DD=ER=?NtA$cf5ebb(L@*H7 z9aPjy!k9%JAX{3oq29>Z>Dq*cxOm-4emLJrENMi0FT53VCfZn?%fdlEgq+D;Gv_%f zb1pA2$Ym^P3_|^?g}R|mTWZ+Eg1X#7-B4FEWCkx%U6$wsj3BPIId($5#X@PQ2u8c_ z97Sr1b)SZM3`2)#;x;vX6fq+t;{mBSKxct)1MEP6u+qNVRm(M83&W{4;0;g_pZ2xZ zM!PB|LPf}ih!jFen8Hv?_j7e74fE^y7^;m%D*{?+Ls;)-=8V=m+0i&_m4XU~bhp!W z`@^O)7hN63cBY$x5h4NuiU~XQ2&H2O4Z;T_TQ0Z5*9YTdlA4P?57~7o^Ux z#7?Mv7FrLrJ`VRx(v717aU5-Z^*}|=I*txoAIO}#Bh>m#-7fM17kaYIAqDk#3#Fl= zF~+J(6?6pBq#I^F+q~1-I$h+aoFfX7@TKw|raZ@TB5g~uSMIuNg8C{$W~3o?#&QhFZm6>vGDI13ncX~MbLfN$!?Ei66+W|jtyu@u zIy0wNxw)-P$c?|6j`MWQ*1_y~4nW_RgpSIfz}ump%C(?*(CZ>?k25nG>}JIkdLT+t zJxMpjaVjN&m@)q?OFFZHI<46pqEk<($1!BO5K?cC zi87WG+3nk``xMk`E!6!{RzDp>(ou>J^}cyZ@}}LcBj;@E`9n?avnHSdx}dgC0<<3L za)wNuAoY!y2=(7A)CKi93pq0lf;iO%(Ft|dLC zt)diER0^F{^>BQBHqI1O6rWf(@KKaK>72F70q=|hM?Me5A;bkW+P9hgV{6rO2(z!W zRy|Pfw$O!8kq140fNY*(mW~n?g~Dh|mL}oRg$FsDsd0wVdtEEQc|t4P;W$d6663aW zroT%$+qiJf+4d)yatlkE+5XsT>eN~AB{$)08kIiH7We`bcux$;)`L|-HN{rWq%wwk z(~&GG^v8UmgL{PSpUx#KY9Gz|MyD94^QYph@AhSV7tek7u%z_>>Z2CwhWaCh^r-F{ zMe3QD2=!SDbwT}2gRQJ&#PL0A*a`I!3#FkVXEVDX^>j>xip=RtLa6m|xM%q-2bsuL zu%!Bc`lN-{Lw%YdQy)mZ{}AIN;SnadUu}JLLfvbj4N&3G6cee~y>mu8z&>i$SL$L` zua6<9pR`aK>dh8PLH&k>(uiQcg;G$P4@Q~O%UE5-GWu~GBswiG1@(g#N<+QQLMf>C zS||lauGLj2TO97%6Qy?o3b6laW}*{ zp$5p^QeSl7UgD2#Cm%vpk-O`B?!dj<@9vj;Xdg>*H{0h9+~51%DITbk?11fpy23&m zpsr-d3^JtpVj|Rj3w1#qXs|`<3zoP6>K7Y`rXB14^3mzmuoLPM3w1+9h-TO#)e{q; zA|69T>N7DBYJGI>oQ3K{Nc8Oj)MeJ!dZ>s;XWB7}RBud#iaMmZ6DmS9wn%+0wuM?B zu=}ls8}M2L+-7~Hp&~?`=~^4``(h$gL`mg`S|7(+XPBSOK9B?5=|Guk)_n@<9TrMM zh5Lj@uRg-;Tdb8c=68%WJ?vqYwAMh)Hkm`Tgh1WMkgkTVQKSyDQ?YYJ0rx+$?mMA| z*8K*kaH!pPCb)Oj8rOXX+z(jyJy7f2cOU0aGq0X>S z3hKEQN+X^tEtG=#2@9onv-%jzXemQtkL9JH9y$)qmWDdTLMf;VEtH0Oo`q6SFR_rb zl-NQU>m@-`!o~P08+ONn;r5Pc769^+|@z7)I(3F%jxt zEYt<{uMM^)*Z42u*li6vp*~@u^-z(s8SF^C7!#o)b6NwS*2m$lMQ$7&h~shVJ`EK) z>o___k$OHRLPh2@wL`6s!(C$BIMyPLpIi4Epdx1-$66c5OED2DGN*^3q1MOYE-`K# z>k!8;tot6Q^>M7Tal9N8p(67!Y8}-2IMz7>4LxNEE7VV+{>(z_p+3cs>8D72$WF!1 zS_AhhtglX}PgG!go0#v&pTh(i;7ALLHW3v~~3>1z$B$Xb_k-6&GNIma7UO+ zW0PypiXqMkH9+pXYbv;x_@mp&hfr1I&by|9d$-@+FZs|umgLU6rh@x>zq?+(O#V=O zxeWDo3pow=6+ZN!rCeaEj}=LwIK8hj|AO5dGTs-fhfa%g+oEUbhS}ogG;Ti6x`nuL(r$F;neD#JQa+(8`FP~^Ip*(UiAQiHACH|r%JCe` z%lu(1TUg@BPifz})F5~UW-+IivXuNk0bgd0G_k~8ddZjf%HPNQfTiS5INl(~uGiMr!IN-PMZ3aK(G$Zj2Gv4{duhioG z8oW%1_lM-G%XonXZxYG(m{AVA+9hASlW+at#Wd`xB6qmJj#}3)F!9Tm7Uj+s*m7dg zy9^&Mv*F!nyr^T2#(#bFyZbGMgI^@TR#w>V1;3*B_xk_d0sP}9&flzm{JOi0H!i8s zzK!$^MW2h<)2+e6MmjeDTTr|F80! z@KzJa(Jb*};wQ8k9Pu0|Kf{jyHnaU?mQz?xWjUSY2`rWP$o9BVw1=#JVXpp#`QK%U zU&Tgyp2ctduV#s#$VPige_OeqoIA@LK|k5X>Fq49V7Y62AO)i?SP)@$^ek8%I;^$V<*GXGDp-GeND$nwQg3=``=)&;`umJ(qf7GK~-_g%}{Tk~(c5=k}k9igAy{w;?xjy!@Jp5$C zXk{t4lD~@U4ZoM;)ZbiVkMkjvZ`XW-|A6HqEamp_C_nZp#rX|(ZuOP_S?=%8vBYlF zFK`<5^S72C8>WBu?Zy)O4xoJh-{PMl+V3Ead%TDro3_6$+Dq0C`UCvJ54K5{?ek{s zCv!a5l>N>6!-nbi^1Sm+mgpZc{{!_u$;VGiv~oYk7TxIga(=-0$6n6ZlN;yz(m%GL zmh%a=u$KIv@k{o&oGJCGGzJRGYl@TQ*mB~eB?TWcE{G9xQ>?R;TR`aN9B1I zu0t@7;yeoLj69FRc|F!0T&I~*#D9I|y^qU{^AO~Jp!{ObM)UdK$o?h&3dV0{c`M7? zSt{|lmCu{L$`ZR8B3?mP&3yemlO?t-#1@B=k9xp%YiR#>@%j6?EU|r|+;|QB1AA#5 zsDH@sC1ih)?SG*5{b{Qy(Pr+y*pmp?r7}L5zsyh8&!Hz6WHQU6Ssu?)ZqOv{=P|FH zilL`bR&+d~~3Gh<=CpNzOmg|6^QE_&*ST9?Qxgu?87niJMor(Ixro+3!Y{xI1+frzQX6jNir*x1YZ2;qPXA4@=x%!o4WzzuaVy z|75v+oJr$BD#`y>=6#DLo~MDmnsVwox z%X>ZiZpL#g@k|Szf06#5J;)%bgN^(%OFVTf`Lp==JeGKL7WR@qhwnX|%M#DV;yGH$ zU(bFwvc$u$S8-bMKhF4VEb(;fcRl>wjPGHI$5-*FtMrfif-U3CkrYeZowPNTJoP~ z{B@Riw*l`#NIv4kyF`09ZoIr8`3FolsWIxEJUP{qo77mIS_h{9`ejre%A>#%yn==3~hZ36#OO_`MeR+*C##W668{J_*q4MKsml3Q=RlL1qDB(&qty09{Ajgf zQ<9%!)b^ny{8**RN%&ONnut$xV$?vBN~p{TpYGxFog8trN%5Tt2j@Em4&EvdlWbBa zIJjOL$EiQ;mj`q=8%ynulMKH0R!1?89oG1}`^dBAW1F9DcBsz{HI9vP-pb=+5#eLh zRJHx0)`-b-pjyTJTNg&Kx=aU1pZC-M$p7Lfsa`iiZ3Mo=0DLa0woy|1FSCEFfmbkI z5co&gPm2AZ#&2gF+h_b2rS<2_^k2u*%Qv}~N2pS}7{{IfUuXQ^8OM9enx{Wc(El-1 ztLV48G)SLk>A#L2Ki7fyUtoMt#IGAr^Lqt8f$C4=BLY7z!jDxg+@In9B02<$hkGiq-t~)dpY1 z_+Jsvc?Lh9()#18w`Q^%qmEF295lT8ze#aUkMcRvnI}(={J8VseCEsfc4dUS^Y;44 zk2`->2tR_#`zX)d4$(TZS+rg@H;*D{T}==5B^&ZKAujhk8s0L#}nR~fS2gx zw^h08!UB(-$A(USEq3Q#5%6u4Nda%6A!+R z@T76~nup&)7iCHQPx9ak2%kLmST$aZBmK^2lAj(Aev$T5cYaIF8hy5V`1+z#&xgG{ zABN2Tw12Y7DqhDq9950#xmoStb@?!%06HEsB;yx+P$)0j;Vn4Haynm77yu{;YhX>#3!N2LjAMxN%M>x$t zohA@{-l;#YdiaOa07)wE$sYV155C5OX9(B*{8-V?hdlhN2ycxdZv1nbhyS$*r}KO+ z?^oFDVGsW)53XNV(c@z8`NjdC9;pd*el?>plqhve)F0gI*;&L-s?T?O@be>_;_on- z{DsDY{)8U>Ru6sy;dVy!K!O66J*UZncMfqbzVY{}OK zRklznhCwmhQmm@%V3^$$>C!0*Wt6>X?Hl(@ZX> z(R`_>Oi;yAtukE9DhfN|SKH>#n=@Y(G8b>v+H5skG^L$CZ}tLp(QrPjX?%`~d}a!T z(w3l98w@M#ZzwaOijE1 z)3UV-*DMW|p0^|jDAnLAr-CIHp0{w#%EgUGsx@j(f|j4x6)auB9#$+_tAgdL&tJ4~ zb#VT&W$C4zLFd9ntCwo8g?wK&$X17=Qt9>#i<{fqm#<#AXmN06+nH_eP}Tf^uJfTx zbyMSk{z@1&B9$;JjKgAXs7O9*jl>`bE0tm?D3r1p-Cr7yQU@z$hsup$u(ESa5H}@l z;H(EjXQrI*tI_v5{BiK*bmMV={)FB$7Afy(^DAP1&HXiFIepQrPclu(k zkw;Zo-a-Sa5v52Ap(t+D0UBx1@NI+w8h05Q@!7#mL4PJ+h>yRB zR6y}%3gt{~P!~CNGEY^*T2sN#Ogwi^4;)j$&fwXv?$Xu$N3=y+wc zx?{Qaw&4yU#wo$bCfpDl?+e%T(00xFiMH2uG@Zp33Rw{?Zgpg{b zy4L5iR7=5NrkK-%AwNL#W5LOfB5W8WbLOdF<@uC-WkpAC%8WJ(2^ z>GWixBW5*atlH?D zC~RA}Xyxg(%mAkbioC|ARI;h%mTZ`n+$Xdl5*&J4?`UESYaU!5& zm9UV}2CS6}HPsdk*fvtz21=Byg(LJ=pS=*N6s>S=;UJ%{4CcrQQpVJrxEKrQ+Dhj| z4x+d+L-{NzdbMn$!Kd2j3~ET9aJJ>bzTp8f%M=Gf!c8(??ALbYP+wmq+>BHqUkssB zl|iNUfXbfCsrdiuufF3lhW_(uy}jRu`+oAyi{{pUW>SJK-K;(C3qrQmlS#+*Kj1l} z)Z5oM5XcijWGU^J5dNqA`%8!)_ZcB)vmIToM<2S3YWV1{lPA)D*yDaABsQ9&%c-cI zU{Bu!MIU|NhQ5-r_PB2eIZ+?1`^ft1p<}cyuZgxo~OU$3*reNf0U z+spVFRu_|^>lFTQzZCKwK7jmX`9I9|i`XIVt3rks6|Nw3cz_j4f;Z)8M&kcPZj*yFx0H6N;z)~Bp+$WpAhZvI(yvb9(j}YPm!%IKk~1Cu`bK9~NW8W1=7!A*3mA^)LPO{e?pkx+vof7nC5OtgA?`~7s; z*iSLS{)HWY*GSPA;>Z1tDQrK;@q^zAh2>Z}t|_wx4*Syt4tJ0f*dl+^A zrg`k|WW$-4gK4C+pXIUt@e78(?$9^3*AMwgpW5QozZ$!FN4~LrJKM|p>p9$bZD~!m zN1m(LzLBL=@eITG!7YtPU(&root_hash), hasher::HASH_SIZE); + str_root_hash.swap(ctx.curr_hash_state); + + if (!ctx.cache.empty()) + { + ctx.prev_hash_state = ctx.cache.rbegin()->second.state; + } + else + { + ctx.prev_hash_state = ctx.curr_hash_state; + } + + ctx.state_syncing_thread = std::thread([&] { + run_state_sync_iterator(); + LOG_ERR << "Exit state sync thread\n"; + exit(1); + }); + ctx.prev_close_time = util::get_epoch_milliseconds(); return 0; } @@ -46,7 +74,6 @@ int init() void consensus() { // A consensus round consists of 4 stages (0,1,2,3). - // For a given stage, this function may get visited multiple times due to time-wait conditions. // Get the latest current time. @@ -145,9 +172,17 @@ void consensus() } if (is_lcl_desync) { + int64_t diff = 0; + if (time_off > ctx.time_now) + diff = time_off - ctx.time_now; + + else + diff = ctx.time_now - time_off; //We are resetting to stage 0 to avoid possible deadlock situations by resetting every node in random time using max time. //this might not make sense now after stage 1 now since we are applying a stage time resolution?. - timewait_stage(true, time_off - ctx.time_now); + + LOG_DBG << "time off: " << std::to_string(diff); + timewait_stage(true, diff); //LOG_DBG << "time off: " << std::to_string(time_off); return; } @@ -157,17 +192,23 @@ void consensus() conf::change_operating_mode(conf::OPERATING_MODE::PROPOSING); } - // In stage 1, 2, 3 we vote for incoming proposals and promote winning votes based on thresholds. - const p2p::proposal stg_prop = create_stage123_proposal(votes); - broadcast_proposal(stg_prop); + if (ctx.stage == 1 || (ctx.stage == 3 && ctx.is_state_syncing)) + check_state(votes); - if (ctx.stage == 3) + if (!ctx.is_state_syncing) { - ctx.prev_close_time = stg_prop.time; - apply_ledger(stg_prop); + // In stage 1, 2, 3 we vote for incoming proposals and promote winning votes based on thresholds. + const p2p::proposal stg_prop = create_stage123_proposal(votes); + broadcast_proposal(stg_prop); - // We have finished a consensus round (all 4 stages). - LOG_INFO << "****Stage 3 consensus reached****"; + if (ctx.stage == 3) + { + ctx.prev_close_time = stg_prop.time; + apply_ledger(stg_prop); + + // We have finished a consensus round (all 4 stages). + LOG_INFO << "****Stage 3 consensus reached****"; + } } } @@ -276,12 +317,13 @@ p2p::proposal create_stage0_proposal() ctx.novel_proposal_time = ctx.time_now; stg_prop.stage = 0; stg_prop.lcl = ctx.lcl; + stg_prop.curr_hash_state = ctx.curr_hash_state; // Populate the proposal with set of candidate user pubkeys. for (const std::string &pubkey : ctx.candidate_users) stg_prop.users.emplace(pubkey); - // We don't need candidate_users anymore, so clear it. It will be repopulated during next censensus round. + // We don't need candidate_users anymore, so clear it. It will be repopulated during next consensus round. ctx.candidate_users.clear(); // Populate the proposal with hashes of user inputs. @@ -292,7 +334,6 @@ p2p::proposal create_stage0_proposal() for (const auto &[hash, cand_output] : ctx.candidate_user_outputs) stg_prop.hash_outputs.emplace(hash); - // todo: set propsal states // todo: generate stg_prop hash and check with ctx.novel_proposal, we are sending same proposal again. return stg_prop; @@ -304,10 +345,11 @@ p2p::proposal create_stage123_proposal(vote_counter &votes) p2p::proposal stg_prop; stg_prop.stage = ctx.stage; - // we always vote for our current lcl regardless of what other peers are saying + // we always vote for our current lcl and state regardless of what other peers are saying // if there's a fork condition we will either request history and state from // our peers or we will halt depending on level of consensus on the sides of the fork stg_prop.lcl = ctx.lcl; + stg_prop.curr_hash_state = ctx.curr_hash_state; // Vote for rest of the proposal fields by looking at candidate proposals. for (const auto &[pubkey, cp] : ctx.candidate_proposals) @@ -330,8 +372,6 @@ p2p::proposal create_stage123_proposal(vote_counter &votes) for (const std::string &hash : cp.hash_outputs) if (ctx.candidate_user_outputs.count(hash) > 0) increment(votes.outputs, hash); - - // todo: repeat above for state } const float_t vote_threshold = get_stage_threshold(ctx.stage); @@ -356,15 +396,13 @@ p2p::proposal create_stage123_proposal(vote_counter &votes) if (numvotes >= vote_threshold) stg_prop.hash_outputs.emplace(hash); - // todo:add states which have votes over stage threshold to proposal. - // time is voted on a simple sorted and majority basis, since there will always be disagreement. - int32_t highest_votes = 0; + int32_t highest_time_vote = 0; for (const auto [time, numvotes] : votes.time) { - if (numvotes > highest_votes) + if (numvotes > highest_time_vote) { - highest_votes = numvotes; + highest_time_vote = numvotes; stg_prop.time = time; } } @@ -393,10 +431,10 @@ void broadcast_proposal(const p2p::proposal &p) p2pmsg::create_msg_from_proposal(msg.builder(), p); p2p::broadcast_message(msg, true); - LOG_DBG << "Proposed [stage" << std::to_string(p.stage) - << "] users:" << p.users.size() - << " hinp:" << p.hash_inputs.size() - << " hout:" << p.hash_outputs.size(); + // LOG_DBG << "Proposed [stage" << std::to_string(p.stage) + // << "] users:" << p.users.size() + // << " hinp:" << p.hash_inputs.size() + // << " hout:" << p.hash_outputs.size(); } /** @@ -452,7 +490,7 @@ void check_lcl_votes(bool &is_desync, bool &should_request_history, uint64_t &ti } //keep track of max time of peers, so we can reset nodes in a random time range to increase reliability. - //This is very usefull especially boostrapping a node cluster. + //This is very useful especially boostrapping a node cluster. if (cp.time > time_off) time_off = cp.time; } @@ -460,9 +498,9 @@ void check_lcl_votes(bool &is_desync, bool &should_request_history, uint64_t &ti is_desync = false; should_request_history = false; - if (total_lcl_votes < (0.8 * conf::cfg.unl.size())) + if (total_lcl_votes < (MAJORITY_THRESHOLD * conf::cfg.unl.size())) { - LOG_DBG << "Not enough peers proposing to perform consensus" << std::to_string(total_lcl_votes) << " needed " << std::to_string(0.8 * conf::cfg.unl.size()); + LOG_DBG << "Not enough peers proposing to perform consensus" << std::to_string(total_lcl_votes) << " needed " << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size()); is_desync = true; //Not enough nodes are propsing. So Node is switching to Proposing if it's in observing mode. @@ -496,7 +534,7 @@ void check_lcl_votes(bool &is_desync, bool &should_request_history, uint64_t &ti return; } - if (winning_votes < 0.8 * ctx.candidate_proposals.size()) + if (winning_votes < MAJORITY_THRESHOLD * ctx.candidate_proposals.size()) { // potential fork condition. LOG_WARN << "No consensus on lcl. Possible fork condition. " << std::to_string(winning_votes) << std::to_string(ctx.candidate_proposals.size()); @@ -580,6 +618,7 @@ void apply_ledger(const p2p::proposal &cons_prop) const std::tuple new_lcl = save_ledger(cons_prop); ctx.led_seq_no = std::get<0>(new_lcl); ctx.lcl = std::get<1>(new_lcl); + ctx.prev_hash_state = ctx.curr_hash_state; // After the current ledger seq no is updated, we remove any newly expired inputs from candidate set. { @@ -596,9 +635,6 @@ void apply_ledger(const p2p::proposal &cons_prop) // Send any output from the previous consensus round to locally connected users. dispatch_user_outputs(cons_prop); - // todo:check state against the winning / canonical state - // and act accordingly (rollback, ask state from peer, etc.) - // This will hold a list of file blocks that was updated by the contract process. // We then feed this information to state tracking logic. proc::contract_fblockmap_t updated_blocks; @@ -663,6 +699,65 @@ void dispatch_user_outputs(const p2p::proposal &cons_prop) } } +/** + * Check state against the winning and canonical state + * @param cons_prop The proposal that achieved consensus. + */ +void check_state(vote_counter &votes) +{ + std::string majority_state; + + for (const auto &[pubkey, cp] : ctx.candidate_proposals) + { + increment(votes.state, cp.curr_hash_state); + } + + int32_t winning_votes = 0; + for (const auto [state, votes] : votes.state) + { + if (votes > winning_votes) + { + winning_votes = votes; + majority_state = state; + } + } + + if (ctx.stage == 1 || ctx.stage == 3) + { + if (ctx.is_state_syncing) + { + std::lock_guard lock(cons::ctx.state_syncing_mutex); + hasher::B2H root_hash = hasher::B2H_empty; + int ret = statefs::compute_hash_tree(root_hash); + std::string str_root_hash(reinterpret_cast(&root_hash), hasher::HASH_SIZE); + str_root_hash.swap(ctx.curr_hash_state); + } + } + + if (ctx.stage == 1 && majority_state != ctx.curr_hash_state) + { + if (ctx.state_sync_lcl != ctx.lcl) + { + LOG_DBG << "State mismatch. Starting state sync..."; + + // Change the mode to passive and not sending out proposals till the state is synced + conf::change_operating_mode(conf::OPERATING_MODE::OBSERVING); + + const hasher::B2H majority_state_hash = *reinterpret_cast(majority_state.c_str()); + start_state_sync(majority_state_hash); + + ctx.is_state_syncing = true; + ctx.state_sync_lcl = ctx.lcl; + } + } + else if (majority_state == ctx.curr_hash_state) + { + ctx.is_state_syncing = false; + ctx.state_sync_lcl.clear(); + conf::change_operating_mode(conf::OPERATING_MODE::PROPOSING); + } +} + /** * Transfers consensus-reached inputs into the provided contract buf map so it can be fed into the contract process. * @param bufmap The contract bufmap which needs to be populated with inputs. diff --git a/src/cons/cons.hpp b/src/cons/cons.hpp index 697604a5..aa421331 100644 --- a/src/cons/cons.hpp +++ b/src/cons/cons.hpp @@ -6,6 +6,8 @@ #include "../proc/proc.hpp" #include "../p2p/p2p.hpp" #include "../usr/user_input.hpp" +#include "ledger_handler.hpp" +#include "state_handler.hpp" namespace cons { @@ -72,14 +74,22 @@ struct consensus_context uint64_t time_now; std::string lcl; uint64_t led_seq_no; + std::string curr_hash_state; + std::string prev_hash_state; + //Map of closed ledgers(only lrdgername[sequnece_number-hash], state hash) with sequence number as map key. //contains closed ledgers from latest to latest - MAX_LEDGER_SEQUENCE. //this is loaded when node started and updated throughout consensus - delete ledgers that falls behind MAX_LEDGER_SEQUENCE range. //We will use this to track lcls related logic.- track state, lcl request, response. - std::map lcl_list; + std::map cache; //ledger close time of previous hash uint64_t prev_close_time; + bool is_state_syncing; + std::string state_sync_lcl; + std::thread state_syncing_thread; + std::mutex state_syncing_mutex; + consensus_context() : recent_userinput_hashes(200) { } @@ -93,6 +103,7 @@ struct vote_counter std::map users; std::map inputs; std::map outputs; + std::map state; }; extern consensus_context ctx; @@ -127,6 +138,8 @@ void apply_ledger(const p2p::proposal &proposal); void dispatch_user_outputs(const p2p::proposal &cons_prop); +void check_state(vote_counter &votes); + void feed_user_inputs_to_contract_bufmap(proc::contract_bufmap_t &bufmap, const p2p::proposal &cons_prop); void extract_user_outputs_from_contract_bufmap(proc::contract_bufmap_t &bufmap); diff --git a/src/cons/ledger_handler.cpp b/src/cons/ledger_handler.cpp index 4bb55f76..f0593a5d 100644 --- a/src/cons/ledger_handler.cpp +++ b/src/cons/ledger_handler.cpp @@ -3,6 +3,7 @@ #include "../conf.hpp" #include "../crypto.hpp" #include "../p2p/p2p.hpp" +#include "../fbschema/common_helpers.hpp" #include "../fbschema/ledger_helpers.hpp" #include "../fbschema/p2pmsg_helpers.hpp" #include "ledger_handler.hpp" @@ -59,7 +60,10 @@ const std::tuple save_ledger(const p2p::proposal &p write_ledger(file_name, ledger_str.data(), ledger_str.size()); - cons::ctx.lcl_list.emplace(led_seq_no, file_name); + ledger_cache c; + c.lcl = file_name; + c.state = proposal.curr_hash_state; + cons::ctx.cache.emplace(led_seq_no, std::move(c)); //Remove old ledgers that exceeds max sequence range. if (led_seq_no > MAX_LEDGER_SEQUENCE) @@ -76,7 +80,7 @@ const std::tuple save_ledger(const p2p::proposal &p */ void remove_old_ledgers(const uint64_t led_seq_no) { - std::map::iterator itr; + std::map::iterator itr; std::string dir_path; @@ -84,13 +88,13 @@ void remove_old_ledgers(const uint64_t led_seq_no) dir_path.append(conf::ctx.histdir) .append("/"); - for (itr = cons::ctx.lcl_list.begin(); - itr != cons::ctx.lcl_list.lower_bound(led_seq_no + 1); + for (itr = cons::ctx.cache.begin(); + itr != cons::ctx.cache.lower_bound(led_seq_no + 1); itr++) { - const std::string file_name = itr->second; + const std::string file_name = itr->second.lcl; std::string file_path; - file_path.reserve(dir_path.size() + itr->second.size() + 4); + file_path.reserve(dir_path.size() + itr->second.lcl.size() + 4); file_path.append(dir_path) .append(file_name) .append(".lcl"); @@ -98,8 +102,9 @@ void remove_old_ledgers(const uint64_t led_seq_no) if (boost::filesystem::exists(file_path)) boost::filesystem::remove(file_path); } - if (!cons::ctx.lcl_list.empty()) - cons::ctx.lcl_list.erase(cons::ctx.lcl_list.begin(), cons::ctx.lcl_list.lower_bound(led_seq_no + 1)); + + if (!cons::ctx.cache.empty()) + cons::ctx.cache.erase(cons::ctx.cache.begin(), cons::ctx.cache.lower_bound(led_seq_no + 1)); } /** @@ -154,7 +159,7 @@ const ledger_history load_ledger() for (const auto &entry : boost::filesystem::directory_iterator(conf::ctx.histdir)) { const boost::filesystem::path file_path = entry.path(); - const std::string file_name = entry.path().stem().string(); + const std::string file_name = file_path.stem().string(); if (boost::filesystem::is_directory(file_path)) { @@ -172,7 +177,22 @@ const ledger_history load_ledger() if (pos != std::string::npos) { seq_no = std::stoull(file_name.substr(0, pos)); - ldg_hist.lcl_list.emplace(seq_no, file_name); //lcl -> [seq_no-hash] + + std::ifstream file(file_path.string(), std::ios::binary | std::ios::ate); + std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(size); + if (file.read(buffer.data(), size)) + { + const uint8_t *ledger_buf_ptr = reinterpret_cast(buffer.data()); + const fbschema::ledger::Ledger *ledger = fbschema::ledger::GetLedger(ledger_buf_ptr); + ledger_cache c; + c.lcl = file_name; + c.state = fbschema::flatbuff_bytes_to_sv(ledger->state()); + + ldg_hist.cache.emplace(seq_no, std::move(c)); //lcl_cache -> [seq_no-hash] + } } else { @@ -183,15 +203,15 @@ const ledger_history load_ledger() } //check if there is a saved lcl file -> if no send genesis lcl. - if (ldg_hist.lcl_list.empty()) + if (ldg_hist.cache.empty()) { ldg_hist.led_seq_no = 0; ldg_hist.lcl = "0-genesis"; } else { - ldg_hist.led_seq_no = ldg_hist.lcl_list.rbegin()->first; - ldg_hist.lcl = ldg_hist.lcl_list.rbegin()->second; + ldg_hist.led_seq_no = ldg_hist.cache.rbegin()->first; + ldg_hist.lcl = ldg_hist.cache.rbegin()->second.lcl; //Remove old ledgers that exceeds max sequence range. if (ldg_hist.led_seq_no > MAX_LEDGER_SEQUENCE) @@ -241,15 +261,15 @@ bool check_required_lcl_availability(const p2p::history_request &hr) if (req_seq_no > 0) { - const auto itr = cons::ctx.lcl_list.find(req_seq_no); - if (itr == cons::ctx.lcl_list.end()) + const auto itr = cons::ctx.cache.find(req_seq_no); + if (itr == cons::ctx.cache.end()) { LOG_DBG << "Required lcl peer asked for is not in our lcl cache."; //either this node is also not in consesnsus ledger or other node requesting a lcl that is older than node's current // minimum lcl sequence becuase of maximum ledger history range. return false; } - else if (itr->second != hr.required_lcl) + else if (itr->second.lcl != hr.required_lcl) { LOG_DBG << "Required lcl peer asked for is not in our lcl cache."; //either this node or requesting node is in a fork condition. @@ -281,20 +301,20 @@ const p2p::history_response retrieve_ledger_history(const p2p::history_request & min_seq_no = std::stoull(hr.minimum_lcl.substr(0, pos)); //get required lcl sequence number } - const auto itr = cons::ctx.lcl_list.find(min_seq_no); - if (itr != cons::ctx.lcl_list.end()) //requested minimum lcl is not in our lcl history cache + const auto itr = cons::ctx.cache.find(min_seq_no); + if (itr != cons::ctx.cache.end()) //requested minimum lcl is not in our lcl history cache { min_seq_no = itr->first; //check whether minimum lcl node ask for is same as this node's. //eventhough sequence number are same, lcl hash can be changed if one of node is in a fork condition. - if (hr.minimum_lcl != itr->second) + if (hr.minimum_lcl != itr->second.lcl) { - LOG_DBG << "Invalid minimum ledger. Recieved min hash: " << min_lcl_hash << " Node hash: " << itr->second; + LOG_DBG << "Invalid minimum ledger. Recieved min hash: "<< min_lcl_hash << " Node hash: " << itr->second.lcl; history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; return history_response; } } - else if (min_seq_no > cons::ctx.lcl_list.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. + else if (min_seq_no > cons::ctx.cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. { LOG_DBG << "Invalid minimum ledger. Recieved minimum sequence number is ahead of node current lcl sequence. hash: " << min_lcl_hash; history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; @@ -303,32 +323,32 @@ const p2p::history_response retrieve_ledger_history(const p2p::history_request & else { LOG_DBG << "Minimum lcl peer asked for is not in our lcl cache. Therefore sending from node minimum lcl"; - min_seq_no = cons::ctx.lcl_list.begin()->first; + min_seq_no = cons::ctx.cache.begin()->first; } //LOG_DBG << "history request min seq: " << std::to_string(min_seq_no); //copy current history cache. - std::map - lcl_list = cons::ctx.lcl_list; + std::map led_cache = cons::ctx.cache; - //filter out cache from finalized minimum sequence. - lcl_list.erase( - lcl_list.begin(), - lcl_list.lower_bound(min_seq_no)); + //filter out cache and get raw files here. + led_cache.erase( + led_cache.begin(), + led_cache.lower_bound(min_seq_no)); - //Get raw content of lcls that going to be send. - for (auto &[seq_no, lcl_hash] : lcl_list) + //Get raw content of lcls that going to be send. + for (auto &[seq_no, cache] : led_cache) { p2p::history_ledger ledger; - ledger.lcl = lcl_hash; + ledger.lcl = cache.lcl; + ledger.state = cache.state; std::string path; - path.reserve(conf::ctx.histdir.size() + lcl_hash.size() + 5); + path.reserve(conf::ctx.histdir.size() + cache.lcl.size() + 5); path.append(conf::ctx.histdir) .append("/") - .append(lcl_hash) + .append(cache.lcl) .append(".lcl"); //read lcl file @@ -379,7 +399,7 @@ void handle_ledger_history_response(const p2p::history_response &hr) // This means we are in a fork ledger.Remove/rollback current ledger. // Basically in the long run we'll rolback one by one untill we catch up to valid minimum ledger . remove_ledger(ctx.lcl); - cons::ctx.lcl_list.erase(ctx.lcl_list.rbegin()->first); + cons::ctx.cache.erase(ctx.cache.rbegin()->first); } else { @@ -433,27 +453,34 @@ void handle_ledger_history_response(const p2p::history_response &hr) //Save recieved lcl in file system and update lcl history cache for (auto &[seq_no, ledger] : hr.hist_ledgers) { - auto prev_dup_itr = cons::ctx.lcl_list.find(seq_no); - if (prev_dup_itr != cons::ctx.lcl_list.end()) + auto prev_dup_itr = cons::ctx.cache.find(seq_no); + if (prev_dup_itr != cons::ctx.cache.end()) { - remove_ledger(prev_dup_itr->second); - cons::ctx.lcl_list.erase(prev_dup_itr); + remove_ledger(prev_dup_itr->second.lcl); + cons::ctx.cache.erase(prev_dup_itr); } write_ledger(ledger.lcl, reinterpret_cast(&ledger.raw_ledger[0]), ledger.raw_ledger.size()); - cons::ctx.lcl_list.emplace(seq_no, ledger.lcl); + ledger_cache l; + l.lcl = ledger.lcl; + l.state = ledger.state; + cons::ctx.cache.emplace(seq_no, std::move(l)); } last_requested_lcl = ""; + + const auto latest_lcl_itr = cons::ctx.cache.rbegin(); + cons::ctx.lcl = latest_lcl_itr->second.lcl; + cons::ctx.led_seq_no = latest_lcl_itr->first; - if (cons::ctx.lcl_list.empty()) + if (cons::ctx.cache.empty()) { cons::ctx.led_seq_no = 0; cons::ctx.lcl = "0-genesis"; } else { - const auto latest_lcl_itr = cons::ctx.lcl_list.rbegin(); - cons::ctx.lcl = latest_lcl_itr->second; + const auto latest_lcl_itr = cons::ctx.cache.rbegin(); + cons::ctx.lcl = latest_lcl_itr->second.lcl; cons::ctx.led_seq_no = latest_lcl_itr->first; } } diff --git a/src/cons/ledger_handler.hpp b/src/cons/ledger_handler.hpp index c92e5acd..5da6efc3 100644 --- a/src/cons/ledger_handler.hpp +++ b/src/cons/ledger_handler.hpp @@ -10,13 +10,22 @@ namespace cons //max ledger count constexpr uint64_t MAX_LEDGER_SEQUENCE = 200; +struct ledger_cache +{ + std::string lcl; + std::string state; +}; + +extern ledger_cache cache; + struct ledger_history { std::string lcl; uint64_t led_seq_no; - std::map lcl_list; + std::map cache; }; + extern std::string last_requested_lcl; const std::tuple save_ledger(const p2p::proposal &proposal); @@ -39,6 +48,6 @@ p2p::peer_outbound_message send_ledger_history(const p2p::history_request &hr); void handle_ledger_history_response(const p2p::history_response &hr); -} +} // namespace cons #endif \ No newline at end of file diff --git a/src/cons/state_handler.cpp b/src/cons/state_handler.cpp new file mode 100644 index 00000000..b4f2256c --- /dev/null +++ b/src/cons/state_handler.cpp @@ -0,0 +1,339 @@ +#include +#include "state_handler.hpp" +#include "../fbschema/p2pmsg_helpers.hpp" +#include "../fbschema/p2pmsg_content_generated.h" +#include "../fbschema/common_helpers.hpp" +#include "../p2p/p2p.hpp" +#include "../pchheader.hpp" +#include "../cons/cons.hpp" +#include "../statefs/state_store.hpp" + +namespace cons +{ + +constexpr uint16_t MAX_AWAITING_REQUESTS = 1; +constexpr uint16_t MAX_RESPONSE_WAIT_CYCLES = 100; + +// List of state responses flatbuffer messages to be processed. +std::list candidate_state_responses; + +// List of pending sync requests to be sent out. +std::list pending_requests; + +// List of submitted requests we are awaiting responses for, keyed by expected response hash. +std::unordered_map submitted_requests; + +void request_state_from_peer(const std::string &path, const bool is_file, const std::string &lcl, const int32_t block_id, const hasher::B2H expected_hash) +{ + p2p::state_request sr; + sr.parent_path = path; + sr.is_file = is_file; + sr.block_id = block_id; + sr.expected_hash = expected_hash; + + p2p::peer_outbound_message msg(std::make_unique(1024)); + fbschema::p2pmsg::create_msg_from_state_request(msg.builder(), sr, lcl); + p2p::send_message_to_random_peer(msg); +} + +int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr) +{ + if (sr.block_id > -1) + { + std::vector blocks; + + if (statefs::get_block(blocks, sr.parent_path, sr.block_id, sr.expected_hash) == -1) + return -1; + + p2p::block_response resp; + resp.path = sr.parent_path; + resp.block_id = sr.block_id; + resp.hash = sr.expected_hash; + resp.data = std::string_view(reinterpret_cast(blocks.data()), blocks.size()); + + fbschema::p2pmsg::create_msg_from_block_response(msg.builder(), resp, ctx.lcl); + } + else + { + if (sr.is_file) + { + std::vector existing_block_hashmap; + + if (statefs::get_block_hash_map(existing_block_hashmap, sr.parent_path, sr.expected_hash) == -1) + return -1; + + fbschema::p2pmsg::create_msg_from_filehashmap_response(msg.builder(), sr.parent_path, existing_block_hashmap, statefs::get_file_length(sr.parent_path), sr.expected_hash, ctx.lcl); + } + else + { + std::unordered_map existing_fs_entries; + + if (statefs::get_fs_entry_hashes(existing_fs_entries, sr.parent_path, sr.expected_hash) == -1) + return -1; + + fbschema::p2pmsg::create_msg_from_fsentry_response(msg.builder(), sr.parent_path, existing_fs_entries, sr.expected_hash, ctx.lcl); + } + } + + return 0; +} + +void start_state_sync(const hasher::B2H state_hash_to_request) +{ + std::cout << "start_state_sync() " << state_hash_to_request << "\n"; + + { + std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); + p2p::ctx.collected_msgs.state_response.clear(); + } + + { + std::lock_guard lock(cons::ctx.state_syncing_mutex); + candidate_state_responses.clear(); + pending_requests.clear(); + submitted_requests.clear(); + } + + // Send the root state request. + submit_request(backlog_item{BACKLOG_ITEM_TYPE::DIR, "/", -1, state_hash_to_request}); +} + +int run_state_sync_iterator() +{ + while (true) + { + util::sleep(120); + + // TODO: Also bypass peer session handler responses if not syncing. + if (!ctx.is_state_syncing) + continue; + + { + std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); + + // Move collected state responses over to local candidate responses list. + if (!p2p::ctx.collected_msgs.state_response.empty()) + candidate_state_responses.splice(candidate_state_responses.end(), p2p::ctx.collected_msgs.state_response); + } + + std::lock_guard lock(cons::ctx.state_syncing_mutex); + + for (auto &response : candidate_state_responses) + { + const fbschema::p2pmsg::Content *content = fbschema::p2pmsg::GetContent(response.data()); + const fbschema::p2pmsg::State_Response_Message *resp_msg = content->message_as_State_Response_Message(); + + // Check whether we are actually waiting for this response's hash. If not, ignore it. + hasher::B2H response_hash = fbschema::flatbuff_bytes_to_hash(resp_msg->hash()); + const auto pending_resp_itr = submitted_requests.find(response_hash); + if (pending_resp_itr == submitted_requests.end()) + { + std::cout << "Ignoring state response.\n"; + continue; + } + + // Now that we have received matching hash, remove it from the waiting list. + submitted_requests.erase(pending_resp_itr); + + // Process the message based on response type. + const fbschema::p2pmsg::State_Response msg_type = resp_msg->state_response_type(); + + if (msg_type == fbschema::p2pmsg::State_Response_Fs_Entry_Response) + { + if (handle_fs_entry_response(resp_msg->state_response_as_Fs_Entry_Response()) == -1) + return -1; + } + else if (msg_type == fbschema::p2pmsg::State_Response_File_HashMap_Response) + { + if (handle_file_hashmap_response(resp_msg->state_response_as_File_HashMap_Response()) == -1) + return -1; + } + else if (msg_type == fbschema::p2pmsg::State_Response_Block_Response) + { + if (handle_file_block_response(resp_msg->state_response_as_Block_Response()) == -1) + return -1; + } + } + + candidate_state_responses.clear(); + + // Check for long-awaited responses and re-request them. + for (auto &[hash, request] : submitted_requests) + { + if (request.waiting_cycles < MAX_RESPONSE_WAIT_CYCLES) + { + // Increment counter. + request.waiting_cycles++; + } + else + { + // Reset the counter and re-submit request. + request.waiting_cycles = 0; + std::cout << "Resubmit state request\n"; + submit_request(request); + } + } + + // Check whether we can submit any more requests. + if (!pending_requests.empty() && submitted_requests.size() < MAX_AWAITING_REQUESTS) + { + const uint16_t available_slots = MAX_AWAITING_REQUESTS - submitted_requests.size(); + for (int i = 0; i < available_slots && !pending_requests.empty(); i++) + { + const backlog_item &request = pending_requests.front(); + submit_request(request); + pending_requests.pop_front(); + } + } + } + + return 0; +} + +void submit_request(const backlog_item &request) +{ + std::cout << "Submitting state request. type: " << request.type << " path:" << request.path << " blockid: " << request.block_id << "\n"; + + submitted_requests.try_emplace(request.expected_hash, request); + + const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR; + request_state_from_peer(request.path, is_file, ctx.lcl, request.block_id, request.expected_hash); +} + +int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp) +{ + std::cout << "Recieved state fs entry response\n"; + + std::unordered_map state_fs_entry_list; + fbschema::p2pmsg::flatbuf_statefshashentry_to_statefshashentry(state_fs_entry_list, fs_entry_resp->entries()); + + for (const auto [a, b] : state_fs_entry_list) + std::cout << "Recieved fsentry: " << a << "\n"; + + std::unordered_map existing_fs_entries; + std::string_view root_path_sv = fbschema::flatbuff_str_to_sv(fs_entry_resp->path()); + std::string root_path_str(root_path_sv.data(), root_path_sv.size()); + + if (!statefs::is_dir_exists(root_path_str)) + { + statefs::create_dir(root_path_str); + } + else + { + if (statefs::get_fs_entry_hashes(existing_fs_entries, std::move(root_path_str), hasher::B2H_empty) == -1) + return -1; + } + + // Request more info on fs entries that exist on both sides but are different. + for (const auto &[path, fs_entry] : existing_fs_entries) + { + std::cout << "Existing path :" << path << std::endl; + const auto fs_itr = state_fs_entry_list.find(path); + if (fs_itr != state_fs_entry_list.end()) + { + std::cout << "Existing fs_entry_hash :" << fs_entry.hash << std::endl; + std::cout << "Recieved fs_entry_hash :" << fs_itr->second.hash << std::endl; + if (fs_itr->second.hash != fs_entry.hash) + { + if (fs_entry.is_file) + pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, path, -1, fs_itr->second.hash}); + else + pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, path, -1, fs_itr->second.hash}); + } + + state_fs_entry_list.erase(fs_itr); + } + else + { + // If there was an entry that does not exist on other side, delete it from this node. + if (fs_entry.is_file) + { + if (statefs::delete_file(path) == -1) + return -1; + } + else + { + if (statefs::delete_dir(path) == -1) + return -1; + } + } + } + + // Queue the remaining fs entries (that this node does not have at all) to request. + for (const auto &[path, fs_entry] : state_fs_entry_list) + { + if (fs_entry.is_file) + pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, path, -1, fs_entry.hash}); + else + pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, path, -1, fs_entry.hash}); + } + + return 0; +} + +int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp) +{ + std::string_view path_sv = fbschema::flatbuff_str_to_sv(file_resp->path()); + const std::string path_str(path_sv.data(), path_sv.size()); + + std::cout << "Recieved file hash map of " << path_str << std::endl; + + std::vector existing_block_hashmap; + if (statefs::get_block_hash_map(existing_block_hashmap, path_str, hasher::B2H_empty) == -1) + return -1; + + const hasher::B2H *existing_hashes = reinterpret_cast(existing_block_hashmap.data()); + auto existing_hash_count = existing_block_hashmap.size() / hasher::HASH_SIZE; + + const hasher::B2H *resp_hashes = reinterpret_cast(file_resp->hash_map()->data()); + auto resp_hash_count = file_resp->hash_map()->size() / hasher::HASH_SIZE; + + std::cout << "Reieved file hashmap size :" << file_resp->hash_map()->size() << std::endl; + std::cout << "Existing file hashmap size :" << existing_block_hashmap.size() << std::endl; + + auto insert_itr = pending_requests.begin(); + + for (int block_id = 0; block_id < existing_hash_count; ++block_id) + { + if (block_id >= resp_hash_count) + break; + + if (existing_hashes[block_id] != resp_hashes[block_id]) + { + std::cout << "Mismatch in file block :" << block_id << std::endl; + // Insert at front to give priority to block requests while preserving block order. + pending_requests.insert(insert_itr, backlog_item{BACKLOG_ITEM_TYPE::BLOCK, path_str, block_id, resp_hashes[block_id]}); + } + } + + if (existing_hash_count > resp_hash_count) + { + if (statefs::truncate_file(path_str, file_resp->file_length()) == -1) + return -1; + } + else if (existing_hash_count < resp_hash_count) + { + for (int block_id = existing_hash_count; block_id < resp_hash_count; ++block_id) + { + std::cout << "Missing block: " << block_id << "\n"; + // Insert at front to give priority to block requests while preserving block order. + pending_requests.insert(insert_itr, backlog_item{BACKLOG_ITEM_TYPE::BLOCK, path_str, block_id, resp_hashes[block_id]}); + } + } + + return 0; +} + +int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg) +{ + p2p::block_response block_resp = fbschema::p2pmsg::create_block_response_from_msg(*block_msg); + + std::cout << "Recieved block " << block_resp.block_id << " of " << block_resp.path << "\n"; + + if (statefs::write_block(block_resp.path, block_resp.block_id, block_resp.data.data(), block_resp.data.size()) == -1) + return -1; + + return 0; +} + +} // namespace cons \ No newline at end of file diff --git a/src/cons/state_handler.hpp b/src/cons/state_handler.hpp new file mode 100644 index 00000000..66208a58 --- /dev/null +++ b/src/cons/state_handler.hpp @@ -0,0 +1,52 @@ +#ifndef _HP_CONS_STATE_HANDLER_ +#define _HP_CONS_STATE_HANDLER_ + +#include "../pchheader.hpp" +#include "../p2p/p2p.hpp" +#include "../fbschema/p2pmsg_content_generated.h" +#include "../statefs/hasher.hpp" + +namespace cons +{ + +enum BACKLOG_ITEM_TYPE +{ + DIR = 0, + FILE = 1, + BLOCK = 2 +}; + +// Represents a queued up state sync operation which needs to be performed. +struct backlog_item +{ + BACKLOG_ITEM_TYPE type; + std::string path; + int32_t block_id = -1; // Only relevant if type=BLOCK + hasher::B2H expected_hash; + + // No. of cycles that this item has been waiting in pending state. + // Used by pending_responses list to increase wait count. + int16_t waiting_cycles = 0; +}; + +extern std::list candidate_state_responses; + +int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr); + +void request_state_from_peer(const std::string &path, const bool is_file, const std::string &lcl, const int32_t block_id, const hasher::B2H expected_hash); + +void start_state_sync(const hasher::B2H state_hash_to_request); + +int run_state_sync_iterator(); + +void submit_request(const backlog_item &request); + +int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp); + +int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp); + +int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg); + +} // namespace cons + +#endif \ No newline at end of file diff --git a/src/fbschema/common_helpers.cpp b/src/fbschema/common_helpers.cpp index a9926437..51a6e73e 100644 --- a/src/fbschema/common_helpers.cpp +++ b/src/fbschema/common_helpers.cpp @@ -15,13 +15,29 @@ std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uo } /** - * Returns return string_view from Flat Buffer vector of bytes. + * Returns string_view from Flat Buffer vector of bytes. */ std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector *buffer) { return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); } +/** + * Returns return string_view from Flat Buffer string. + */ +std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer) +{ + return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); +} + +/** + * Returns hash from Flat Buffer vector of bytes. + */ +hasher::B2H flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer) +{ + return *reinterpret_cast(buffer->data()); +} + /** * Returns set from Flatbuffer vector of ByteArrays. */ @@ -58,6 +74,24 @@ sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view s return builder.CreateVector(reinterpret_cast(sv.data()), sv.size()); } +/** + * Returns Flatbuffer string from string_view. + */ +const flatbuffers::Offset +sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv) +{ + return builder.CreateString(sv); +} + +/** + * Returns Flatbuffer bytes vector from hash. + */ +const flatbuffers::Offset> +hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, const hasher::B2H hash) +{ + return builder.CreateVector(reinterpret_cast(&hash), hasher::HASH_SIZE); +} + /** * Returns Flatbuffer vector of ByteArrays from given set of strings. */ diff --git a/src/fbschema/common_helpers.hpp b/src/fbschema/common_helpers.hpp index cae89db2..be323695 100644 --- a/src/fbschema/common_helpers.hpp +++ b/src/fbschema/common_helpers.hpp @@ -4,6 +4,7 @@ #include "../pchheader.hpp" #include #include "common_schema_generated.h" +#include "../statefs/hasher.hpp" namespace fbschema { @@ -17,6 +18,10 @@ std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uo std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector *buffer); +std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer); + +hasher::B2H flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer); + const std::set flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector> *fbvec); @@ -28,6 +33,12 @@ flatbuf_pairvector_to_stringmap(const flatbuffers::Vector> sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv); +const flatbuffers::Offset +sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv); + +const flatbuffers::Offset> +hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, hasher::B2H hash); + const flatbuffers::Offset>> stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set &set); diff --git a/src/fbschema/ledger_helpers.cpp b/src/fbschema/ledger_helpers.cpp index 6d68b55c..e8b78910 100644 --- a/src/fbschema/ledger_helpers.cpp +++ b/src/fbschema/ledger_helpers.cpp @@ -12,7 +12,7 @@ namespace fbschema::ledger * Create ledger from the given proposal struct. * @param p The proposal struct to be placed in ledger. */ -std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no) +const std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no) { flatbuffers::Offset ledger = ledger::CreateLedger( @@ -20,13 +20,13 @@ std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &bui seq_no, p.time, sv_to_flatbuff_bytes(builder, p.lcl), + sv_to_flatbuff_bytes(builder, p.curr_hash_state), stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), - stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs) - ); + stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs)); builder.Finish(ledger); // Finished building message content to get serialised content. return flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize()); } -} // namespace fbschema +} // namespace fbschema::ledger diff --git a/src/fbschema/ledger_helpers.hpp b/src/fbschema/ledger_helpers.hpp index 4188fcef..4357dc0e 100644 --- a/src/fbschema/ledger_helpers.hpp +++ b/src/fbschema/ledger_helpers.hpp @@ -9,7 +9,7 @@ namespace fbschema::ledger { -std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no); +const std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no); } #endif \ No newline at end of file diff --git a/src/fbschema/ledger_schema.fbs b/src/fbschema/ledger_schema.fbs index 755ac7ea..63f8144a 100644 --- a/src/fbschema/ledger_schema.fbs +++ b/src/fbschema/ledger_schema.fbs @@ -6,6 +6,7 @@ table Ledger { seq_no:uint64; time:uint64; lcl:[ubyte]; + state:[ubyte]; users: [ByteArray]; inputs: [ByteArray]; outputs: [ByteArray]; diff --git a/src/fbschema/ledger_schema_generated.h b/src/fbschema/ledger_schema_generated.h index 6332d8f6..0d19cabd 100644 --- a/src/fbschema/ledger_schema_generated.h +++ b/src/fbschema/ledger_schema_generated.h @@ -20,9 +20,10 @@ struct Ledger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_SEQ_NO = 4, VT_TIME = 6, VT_LCL = 8, - VT_USERS = 10, - VT_INPUTS = 12, - VT_OUTPUTS = 14 + VT_STATE = 10, + VT_USERS = 12, + VT_INPUTS = 14, + VT_OUTPUTS = 16 }; uint64_t seq_no() const { return GetField(VT_SEQ_NO, 0); @@ -42,6 +43,12 @@ struct Ledger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { flatbuffers::Vector *mutable_lcl() { return GetPointer *>(VT_LCL); } + const flatbuffers::Vector *state() const { + return GetPointer *>(VT_STATE); + } + flatbuffers::Vector *mutable_state() { + return GetPointer *>(VT_STATE); + } const flatbuffers::Vector> *users() const { return GetPointer> *>(VT_USERS); } @@ -66,6 +73,8 @@ struct Ledger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyField(verifier, VT_TIME) && VerifyOffset(verifier, VT_LCL) && verifier.VerifyVector(lcl()) && + VerifyOffset(verifier, VT_STATE) && + verifier.VerifyVector(state()) && VerifyOffset(verifier, VT_USERS) && verifier.VerifyVector(users()) && verifier.VerifyVectorOfTables(users()) && @@ -91,6 +100,9 @@ struct LedgerBuilder { void add_lcl(flatbuffers::Offset> lcl) { fbb_.AddOffset(Ledger::VT_LCL, lcl); } + void add_state(flatbuffers::Offset> state) { + fbb_.AddOffset(Ledger::VT_STATE, state); + } void add_users(flatbuffers::Offset>> users) { fbb_.AddOffset(Ledger::VT_USERS, users); } @@ -117,6 +129,7 @@ inline flatbuffers::Offset CreateLedger( uint64_t seq_no = 0, uint64_t time = 0, flatbuffers::Offset> lcl = 0, + flatbuffers::Offset> state = 0, flatbuffers::Offset>> users = 0, flatbuffers::Offset>> inputs = 0, flatbuffers::Offset>> outputs = 0) { @@ -126,6 +139,7 @@ inline flatbuffers::Offset CreateLedger( builder_.add_outputs(outputs); builder_.add_inputs(inputs); builder_.add_users(users); + builder_.add_state(state); builder_.add_lcl(lcl); return builder_.Finish(); } @@ -135,10 +149,12 @@ inline flatbuffers::Offset CreateLedgerDirect( uint64_t seq_no = 0, uint64_t time = 0, const std::vector *lcl = nullptr, + const std::vector *state = nullptr, const std::vector> *users = nullptr, const std::vector> *inputs = nullptr, const std::vector> *outputs = nullptr) { auto lcl__ = lcl ? _fbb.CreateVector(*lcl) : 0; + auto state__ = state ? _fbb.CreateVector(*state) : 0; auto users__ = users ? _fbb.CreateVector>(*users) : 0; auto inputs__ = inputs ? _fbb.CreateVector>(*inputs) : 0; auto outputs__ = outputs ? _fbb.CreateVector>(*outputs) : 0; @@ -147,6 +163,7 @@ inline flatbuffers::Offset CreateLedgerDirect( seq_no, time, lcl__, + state__, users__, inputs__, outputs__); diff --git a/src/fbschema/p2pmsg_content.fbs b/src/fbschema/p2pmsg_content.fbs index a6482d2e..f48e93e3 100644 --- a/src/fbschema/p2pmsg_content.fbs +++ b/src/fbschema/p2pmsg_content.fbs @@ -12,7 +12,7 @@ table UserSubmittedMessageGroup { messages:[UserSubmittedMessage]; } -union Message { NonUnl_Proposal_Message, Proposal_Message, Npl_Message, History_Request_Message, History_Response_Message } //message content type +union Message { NonUnl_Proposal_Message, Proposal_Message, Npl_Message, State_Request_Message, State_Response_Message, History_Request_Message, History_Response_Message } //message content type table Content { message:Message; @@ -28,7 +28,7 @@ table Proposal_Message { //Proposal type message schema users:[ByteArray]; hash_inputs:[ByteArray]; //stage > 0 inputs (hash of stage 0 inputs) hash_outputs:[ByteArray]; //stage > 0 outputs (hash of stage 0 outputs) - state: State; + curr_state_hash: [ubyte]; } table Npl_Message { //NPL type message schema @@ -58,21 +58,46 @@ table HistoryLedgerPair { //A key, value pair of byte[]. } table HistoryLedger { + state:[ubyte]; lcl:[ubyte]; raw_ledger:[ubyte]; } -table StateDifference { //Represent state difference by tracking created,updated and deleted state files. - created: [BytesKeyValuePair]; //list of { fn => hash } - updated: [BytesKeyValuePair]; - deleted: [BytesKeyValuePair]; +table State_Request_Message { //State request message schema + parent_path:string; + is_file:bool; + block_id:int32; + expected_hash:[ubyte]; } -table State { - previous: [ubyte]; // hash of the previous state - current: [ubyte]; // hash of the current state - difference: StateDifference; - patch: [BytesKeyValuePair]; // fn -> bsdiff patch going from previous state to new state +union State_Response{ File_HashMap_Response, Block_Response, Fs_Entry_Response } + +table State_Response_Message{ + state_response:State_Response; + hash:[ubyte]; +} + +table Fs_Entry_Response{ + path: string; + entries: [State_FS_Hash_Entry]; +} + +table File_HashMap_Response{ + path: string; + file_length:uint64; + hash_map:[ubyte]; +} + +table Block_Response{ + path: string; + block_id:uint32; + data: [ubyte]; +} + +table State_FS_Hash_Entry{ + path: string; + is_file: bool; + hash: [ubyte]; } root_type Content; //root type for message content \ No newline at end of file diff --git a/src/fbschema/p2pmsg_content_generated.h b/src/fbschema/p2pmsg_content_generated.h index 07f42c41..d5c2667e 100644 --- a/src/fbschema/p2pmsg_content_generated.h +++ b/src/fbschema/p2pmsg_content_generated.h @@ -31,27 +31,39 @@ struct HistoryLedgerPair; struct HistoryLedger; -struct StateDifference; +struct State_Request_Message; -struct State; +struct State_Response_Message; + +struct Fs_Entry_Response; + +struct File_HashMap_Response; + +struct Block_Response; + +struct State_FS_Hash_Entry; enum Message { Message_NONE = 0, Message_NonUnl_Proposal_Message = 1, Message_Proposal_Message = 2, Message_Npl_Message = 3, - Message_History_Request_Message = 4, - Message_History_Response_Message = 5, + Message_State_Request_Message = 4, + Message_State_Response_Message = 5, + Message_History_Request_Message = 6, + Message_History_Response_Message = 7, Message_MIN = Message_NONE, Message_MAX = Message_History_Response_Message }; -inline const Message (&EnumValuesMessage())[6] { +inline const Message (&EnumValuesMessage())[8] { static const Message values[] = { Message_NONE, Message_NonUnl_Proposal_Message, Message_Proposal_Message, Message_Npl_Message, + Message_State_Request_Message, + Message_State_Response_Message, Message_History_Request_Message, Message_History_Response_Message }; @@ -64,6 +76,8 @@ inline const char * const *EnumNamesMessage() { "NonUnl_Proposal_Message", "Proposal_Message", "Npl_Message", + "State_Request_Message", + "State_Response_Message", "History_Request_Message", "History_Response_Message", nullptr @@ -93,6 +107,14 @@ template<> struct MessageTraits { static const Message enum_value = Message_Npl_Message; }; +template<> struct MessageTraits { + static const Message enum_value = Message_State_Request_Message; +}; + +template<> struct MessageTraits { + static const Message enum_value = Message_State_Response_Message; +}; + template<> struct MessageTraits { static const Message enum_value = Message_History_Request_Message; }; @@ -137,6 +159,61 @@ inline const char *EnumNameLedger_Response_Error(Ledger_Response_Error e) { return EnumNamesLedger_Response_Error()[index]; } +enum State_Response { + State_Response_NONE = 0, + State_Response_File_HashMap_Response = 1, + State_Response_Block_Response = 2, + State_Response_Fs_Entry_Response = 3, + State_Response_MIN = State_Response_NONE, + State_Response_MAX = State_Response_Fs_Entry_Response +}; + +inline const State_Response (&EnumValuesState_Response())[4] { + static const State_Response values[] = { + State_Response_NONE, + State_Response_File_HashMap_Response, + State_Response_Block_Response, + State_Response_Fs_Entry_Response + }; + return values; +} + +inline const char * const *EnumNamesState_Response() { + static const char * const names[] = { + "NONE", + "File_HashMap_Response", + "Block_Response", + "Fs_Entry_Response", + nullptr + }; + return names; +} + +inline const char *EnumNameState_Response(State_Response e) { + if (e < State_Response_NONE || e > State_Response_Fs_Entry_Response) return ""; + const size_t index = static_cast(e); + return EnumNamesState_Response()[index]; +} + +template struct State_ResponseTraits { + static const State_Response enum_value = State_Response_NONE; +}; + +template<> struct State_ResponseTraits { + static const State_Response enum_value = State_Response_File_HashMap_Response; +}; + +template<> struct State_ResponseTraits { + static const State_Response enum_value = State_Response_Block_Response; +}; + +template<> struct State_ResponseTraits { + static const State_Response enum_value = State_Response_Fs_Entry_Response; +}; + +bool VerifyState_Response(flatbuffers::Verifier &verifier, const void *obj, State_Response type); +bool VerifyState_ResponseVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector> *values, const flatbuffers::Vector *types); + struct UserSubmittedMessage FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_CONTENT = 4, @@ -302,6 +379,12 @@ struct Content FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { const Npl_Message *message_as_Npl_Message() const { return message_type() == Message_Npl_Message ? static_cast(message()) : nullptr; } + const State_Request_Message *message_as_State_Request_Message() const { + return message_type() == Message_State_Request_Message ? static_cast(message()) : nullptr; + } + const State_Response_Message *message_as_State_Response_Message() const { + return message_type() == Message_State_Response_Message ? static_cast(message()) : nullptr; + } const History_Request_Message *message_as_History_Request_Message() const { return message_type() == Message_History_Request_Message ? static_cast(message()) : nullptr; } @@ -332,6 +415,14 @@ template<> inline const Npl_Message *Content::message_as() const { return message_as_Npl_Message(); } +template<> inline const State_Request_Message *Content::message_as() const { + return message_as_State_Request_Message(); +} + +template<> inline const State_Response_Message *Content::message_as() const { + return message_as_State_Response_Message(); +} + template<> inline const History_Request_Message *Content::message_as() const { return message_as_History_Request_Message(); } @@ -432,7 +523,7 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_USERS = 8, VT_HASH_INPUTS = 10, VT_HASH_OUTPUTS = 12, - VT_STATE = 14 + VT_CURR_STATE_HASH = 14 }; uint8_t stage() const { return GetField(VT_STAGE, 0); @@ -464,11 +555,11 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { flatbuffers::Vector> *mutable_hash_outputs() { return GetPointer> *>(VT_HASH_OUTPUTS); } - const State *state() const { - return GetPointer(VT_STATE); + const flatbuffers::Vector *curr_state_hash() const { + return GetPointer *>(VT_CURR_STATE_HASH); } - State *mutable_state() { - return GetPointer(VT_STATE); + flatbuffers::Vector *mutable_curr_state_hash() { + return GetPointer *>(VT_CURR_STATE_HASH); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && @@ -483,8 +574,8 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyOffset(verifier, VT_HASH_OUTPUTS) && verifier.VerifyVector(hash_outputs()) && verifier.VerifyVectorOfTables(hash_outputs()) && - VerifyOffset(verifier, VT_STATE) && - verifier.VerifyTable(state()) && + VerifyOffset(verifier, VT_CURR_STATE_HASH) && + verifier.VerifyVector(curr_state_hash()) && verifier.EndTable(); } }; @@ -507,8 +598,8 @@ struct Proposal_MessageBuilder { void add_hash_outputs(flatbuffers::Offset>> hash_outputs) { fbb_.AddOffset(Proposal_Message::VT_HASH_OUTPUTS, hash_outputs); } - void add_state(flatbuffers::Offset state) { - fbb_.AddOffset(Proposal_Message::VT_STATE, state); + void add_curr_state_hash(flatbuffers::Offset> curr_state_hash) { + fbb_.AddOffset(Proposal_Message::VT_CURR_STATE_HASH, curr_state_hash); } explicit Proposal_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { @@ -529,10 +620,10 @@ inline flatbuffers::Offset CreateProposal_Message( flatbuffers::Offset>> users = 0, flatbuffers::Offset>> hash_inputs = 0, flatbuffers::Offset>> hash_outputs = 0, - flatbuffers::Offset state = 0) { + flatbuffers::Offset> curr_state_hash = 0) { Proposal_MessageBuilder builder_(_fbb); builder_.add_time(time); - builder_.add_state(state); + builder_.add_curr_state_hash(curr_state_hash); builder_.add_hash_outputs(hash_outputs); builder_.add_hash_inputs(hash_inputs); builder_.add_users(users); @@ -547,10 +638,11 @@ inline flatbuffers::Offset CreateProposal_MessageDirect( const std::vector> *users = nullptr, const std::vector> *hash_inputs = nullptr, const std::vector> *hash_outputs = nullptr, - flatbuffers::Offset state = 0) { + const std::vector *curr_state_hash = nullptr) { auto users__ = users ? _fbb.CreateVector>(*users) : 0; auto hash_inputs__ = hash_inputs ? _fbb.CreateVector>(*hash_inputs) : 0; auto hash_outputs__ = hash_outputs ? _fbb.CreateVector>(*hash_outputs) : 0; + auto curr_state_hash__ = curr_state_hash ? _fbb.CreateVector(*curr_state_hash) : 0; return fbschema::p2pmsg::CreateProposal_Message( _fbb, stage, @@ -558,7 +650,7 @@ inline flatbuffers::Offset CreateProposal_MessageDirect( users__, hash_inputs__, hash_outputs__, - state); + curr_state_hash__); } struct Npl_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { @@ -812,9 +904,16 @@ inline flatbuffers::Offset CreateHistoryLedgerPair( struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_LCL = 4, - VT_RAW_LEDGER = 6 + VT_STATE = 4, + VT_LCL = 6, + VT_RAW_LEDGER = 8 }; + const flatbuffers::Vector *state() const { + return GetPointer *>(VT_STATE); + } + flatbuffers::Vector *mutable_state() { + return GetPointer *>(VT_STATE); + } const flatbuffers::Vector *lcl() const { return GetPointer *>(VT_LCL); } @@ -829,6 +928,8 @@ struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_STATE) && + verifier.VerifyVector(state()) && VerifyOffset(verifier, VT_LCL) && verifier.VerifyVector(lcl()) && VerifyOffset(verifier, VT_RAW_LEDGER) && @@ -840,6 +941,9 @@ struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { struct HistoryLedgerBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; + void add_state(flatbuffers::Offset> state) { + fbb_.AddOffset(HistoryLedger::VT_STATE, state); + } void add_lcl(flatbuffers::Offset> lcl) { fbb_.AddOffset(HistoryLedger::VT_LCL, lcl); } @@ -860,218 +964,561 @@ struct HistoryLedgerBuilder { inline flatbuffers::Offset CreateHistoryLedger( flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset> state = 0, flatbuffers::Offset> lcl = 0, flatbuffers::Offset> raw_ledger = 0) { HistoryLedgerBuilder builder_(_fbb); builder_.add_raw_ledger(raw_ledger); builder_.add_lcl(lcl); + builder_.add_state(state); return builder_.Finish(); } inline flatbuffers::Offset CreateHistoryLedgerDirect( flatbuffers::FlatBufferBuilder &_fbb, + const std::vector *state = nullptr, const std::vector *lcl = nullptr, const std::vector *raw_ledger = nullptr) { + auto state__ = state ? _fbb.CreateVector(*state) : 0; auto lcl__ = lcl ? _fbb.CreateVector(*lcl) : 0; auto raw_ledger__ = raw_ledger ? _fbb.CreateVector(*raw_ledger) : 0; return fbschema::p2pmsg::CreateHistoryLedger( _fbb, + state__, lcl__, raw_ledger__); } -struct StateDifference FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { +struct State_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_CREATED = 4, - VT_UPDATED = 6, - VT_DELETED = 8 + VT_PARENT_PATH = 4, + VT_IS_FILE = 6, + VT_BLOCK_ID = 8, + VT_EXPECTED_HASH = 10 }; - const flatbuffers::Vector> *created() const { - return GetPointer> *>(VT_CREATED); + const flatbuffers::String *parent_path() const { + return GetPointer(VT_PARENT_PATH); } - flatbuffers::Vector> *mutable_created() { - return GetPointer> *>(VT_CREATED); + flatbuffers::String *mutable_parent_path() { + return GetPointer(VT_PARENT_PATH); } - const flatbuffers::Vector> *updated() const { - return GetPointer> *>(VT_UPDATED); + bool is_file() const { + return GetField(VT_IS_FILE, 0) != 0; } - flatbuffers::Vector> *mutable_updated() { - return GetPointer> *>(VT_UPDATED); + bool mutate_is_file(bool _is_file) { + return SetField(VT_IS_FILE, static_cast(_is_file), 0); } - const flatbuffers::Vector> *deleted() const { - return GetPointer> *>(VT_DELETED); + int32_t block_id() const { + return GetField(VT_BLOCK_ID, 0); } - flatbuffers::Vector> *mutable_deleted() { - return GetPointer> *>(VT_DELETED); + bool mutate_block_id(int32_t _block_id) { + return SetField(VT_BLOCK_ID, _block_id, 0); + } + const flatbuffers::Vector *expected_hash() const { + return GetPointer *>(VT_EXPECTED_HASH); + } + flatbuffers::Vector *mutable_expected_hash() { + return GetPointer *>(VT_EXPECTED_HASH); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && - VerifyOffset(verifier, VT_CREATED) && - verifier.VerifyVector(created()) && - verifier.VerifyVectorOfTables(created()) && - VerifyOffset(verifier, VT_UPDATED) && - verifier.VerifyVector(updated()) && - verifier.VerifyVectorOfTables(updated()) && - VerifyOffset(verifier, VT_DELETED) && - verifier.VerifyVector(deleted()) && - verifier.VerifyVectorOfTables(deleted()) && + VerifyOffset(verifier, VT_PARENT_PATH) && + verifier.VerifyString(parent_path()) && + VerifyField(verifier, VT_IS_FILE) && + VerifyField(verifier, VT_BLOCK_ID) && + VerifyOffset(verifier, VT_EXPECTED_HASH) && + verifier.VerifyVector(expected_hash()) && verifier.EndTable(); } }; -struct StateDifferenceBuilder { +struct State_Request_MessageBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - void add_created(flatbuffers::Offset>> created) { - fbb_.AddOffset(StateDifference::VT_CREATED, created); + void add_parent_path(flatbuffers::Offset parent_path) { + fbb_.AddOffset(State_Request_Message::VT_PARENT_PATH, parent_path); } - void add_updated(flatbuffers::Offset>> updated) { - fbb_.AddOffset(StateDifference::VT_UPDATED, updated); + void add_is_file(bool is_file) { + fbb_.AddElement(State_Request_Message::VT_IS_FILE, static_cast(is_file), 0); } - void add_deleted(flatbuffers::Offset>> deleted) { - fbb_.AddOffset(StateDifference::VT_DELETED, deleted); + void add_block_id(int32_t block_id) { + fbb_.AddElement(State_Request_Message::VT_BLOCK_ID, block_id, 0); } - explicit StateDifferenceBuilder(flatbuffers::FlatBufferBuilder &_fbb) + void add_expected_hash(flatbuffers::Offset> expected_hash) { + fbb_.AddOffset(State_Request_Message::VT_EXPECTED_HASH, expected_hash); + } + explicit State_Request_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - StateDifferenceBuilder &operator=(const StateDifferenceBuilder &); - flatbuffers::Offset Finish() { + State_Request_MessageBuilder &operator=(const State_Request_MessageBuilder &); + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateStateDifference( +inline flatbuffers::Offset CreateState_Request_Message( flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset>> created = 0, - flatbuffers::Offset>> updated = 0, - flatbuffers::Offset>> deleted = 0) { - StateDifferenceBuilder builder_(_fbb); - builder_.add_deleted(deleted); - builder_.add_updated(updated); - builder_.add_created(created); + flatbuffers::Offset parent_path = 0, + bool is_file = false, + int32_t block_id = 0, + flatbuffers::Offset> expected_hash = 0) { + State_Request_MessageBuilder builder_(_fbb); + builder_.add_expected_hash(expected_hash); + builder_.add_block_id(block_id); + builder_.add_parent_path(parent_path); + builder_.add_is_file(is_file); return builder_.Finish(); } -inline flatbuffers::Offset CreateStateDifferenceDirect( +inline flatbuffers::Offset CreateState_Request_MessageDirect( flatbuffers::FlatBufferBuilder &_fbb, - const std::vector> *created = nullptr, - const std::vector> *updated = nullptr, - const std::vector> *deleted = nullptr) { - auto created__ = created ? _fbb.CreateVector>(*created) : 0; - auto updated__ = updated ? _fbb.CreateVector>(*updated) : 0; - auto deleted__ = deleted ? _fbb.CreateVector>(*deleted) : 0; - return fbschema::p2pmsg::CreateStateDifference( + const char *parent_path = nullptr, + bool is_file = false, + int32_t block_id = 0, + const std::vector *expected_hash = nullptr) { + auto parent_path__ = parent_path ? _fbb.CreateString(parent_path) : 0; + auto expected_hash__ = expected_hash ? _fbb.CreateVector(*expected_hash) : 0; + return fbschema::p2pmsg::CreateState_Request_Message( _fbb, - created__, - updated__, - deleted__); + parent_path__, + is_file, + block_id, + expected_hash__); } -struct State FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { +struct State_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_PREVIOUS = 4, - VT_CURRENT = 6, - VT_DIFFERENCE = 8, - VT_PATCH = 10 + VT_STATE_RESPONSE_TYPE = 4, + VT_STATE_RESPONSE = 6, + VT_HASH = 8 }; - const flatbuffers::Vector *previous() const { - return GetPointer *>(VT_PREVIOUS); + State_Response state_response_type() const { + return static_cast(GetField(VT_STATE_RESPONSE_TYPE, 0)); } - flatbuffers::Vector *mutable_previous() { - return GetPointer *>(VT_PREVIOUS); + bool mutate_state_response_type(State_Response _state_response_type) { + return SetField(VT_STATE_RESPONSE_TYPE, static_cast(_state_response_type), 0); } - const flatbuffers::Vector *current() const { - return GetPointer *>(VT_CURRENT); + const void *state_response() const { + return GetPointer(VT_STATE_RESPONSE); } - flatbuffers::Vector *mutable_current() { - return GetPointer *>(VT_CURRENT); + template const T *state_response_as() const; + const File_HashMap_Response *state_response_as_File_HashMap_Response() const { + return state_response_type() == State_Response_File_HashMap_Response ? static_cast(state_response()) : nullptr; } - const StateDifference *difference() const { - return GetPointer(VT_DIFFERENCE); + const Block_Response *state_response_as_Block_Response() const { + return state_response_type() == State_Response_Block_Response ? static_cast(state_response()) : nullptr; } - StateDifference *mutable_difference() { - return GetPointer(VT_DIFFERENCE); + const Fs_Entry_Response *state_response_as_Fs_Entry_Response() const { + return state_response_type() == State_Response_Fs_Entry_Response ? static_cast(state_response()) : nullptr; } - const flatbuffers::Vector> *patch() const { - return GetPointer> *>(VT_PATCH); + void *mutable_state_response() { + return GetPointer(VT_STATE_RESPONSE); } - flatbuffers::Vector> *mutable_patch() { - return GetPointer> *>(VT_PATCH); + const flatbuffers::Vector *hash() const { + return GetPointer *>(VT_HASH); + } + flatbuffers::Vector *mutable_hash() { + return GetPointer *>(VT_HASH); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && - VerifyOffset(verifier, VT_PREVIOUS) && - verifier.VerifyVector(previous()) && - VerifyOffset(verifier, VT_CURRENT) && - verifier.VerifyVector(current()) && - VerifyOffset(verifier, VT_DIFFERENCE) && - verifier.VerifyTable(difference()) && - VerifyOffset(verifier, VT_PATCH) && - verifier.VerifyVector(patch()) && - verifier.VerifyVectorOfTables(patch()) && + VerifyField(verifier, VT_STATE_RESPONSE_TYPE) && + VerifyOffset(verifier, VT_STATE_RESPONSE) && + VerifyState_Response(verifier, state_response(), state_response_type()) && + VerifyOffset(verifier, VT_HASH) && + verifier.VerifyVector(hash()) && verifier.EndTable(); } }; -struct StateBuilder { +template<> inline const File_HashMap_Response *State_Response_Message::state_response_as() const { + return state_response_as_File_HashMap_Response(); +} + +template<> inline const Block_Response *State_Response_Message::state_response_as() const { + return state_response_as_Block_Response(); +} + +template<> inline const Fs_Entry_Response *State_Response_Message::state_response_as() const { + return state_response_as_Fs_Entry_Response(); +} + +struct State_Response_MessageBuilder { flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - void add_previous(flatbuffers::Offset> previous) { - fbb_.AddOffset(State::VT_PREVIOUS, previous); + void add_state_response_type(State_Response state_response_type) { + fbb_.AddElement(State_Response_Message::VT_STATE_RESPONSE_TYPE, static_cast(state_response_type), 0); } - void add_current(flatbuffers::Offset> current) { - fbb_.AddOffset(State::VT_CURRENT, current); + void add_state_response(flatbuffers::Offset state_response) { + fbb_.AddOffset(State_Response_Message::VT_STATE_RESPONSE, state_response); } - void add_difference(flatbuffers::Offset difference) { - fbb_.AddOffset(State::VT_DIFFERENCE, difference); + void add_hash(flatbuffers::Offset> hash) { + fbb_.AddOffset(State_Response_Message::VT_HASH, hash); } - void add_patch(flatbuffers::Offset>> patch) { - fbb_.AddOffset(State::VT_PATCH, patch); - } - explicit StateBuilder(flatbuffers::FlatBufferBuilder &_fbb) + explicit State_Response_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - StateBuilder &operator=(const StateBuilder &); - flatbuffers::Offset Finish() { + State_Response_MessageBuilder &operator=(const State_Response_MessageBuilder &); + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateState( +inline flatbuffers::Offset CreateState_Response_Message( flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset> previous = 0, - flatbuffers::Offset> current = 0, - flatbuffers::Offset difference = 0, - flatbuffers::Offset>> patch = 0) { - StateBuilder builder_(_fbb); - builder_.add_patch(patch); - builder_.add_difference(difference); - builder_.add_current(current); - builder_.add_previous(previous); + State_Response state_response_type = State_Response_NONE, + flatbuffers::Offset state_response = 0, + flatbuffers::Offset> hash = 0) { + State_Response_MessageBuilder builder_(_fbb); + builder_.add_hash(hash); + builder_.add_state_response(state_response); + builder_.add_state_response_type(state_response_type); return builder_.Finish(); } -inline flatbuffers::Offset CreateStateDirect( +inline flatbuffers::Offset CreateState_Response_MessageDirect( flatbuffers::FlatBufferBuilder &_fbb, - const std::vector *previous = nullptr, - const std::vector *current = nullptr, - flatbuffers::Offset difference = 0, - const std::vector> *patch = nullptr) { - auto previous__ = previous ? _fbb.CreateVector(*previous) : 0; - auto current__ = current ? _fbb.CreateVector(*current) : 0; - auto patch__ = patch ? _fbb.CreateVector>(*patch) : 0; - return fbschema::p2pmsg::CreateState( + State_Response state_response_type = State_Response_NONE, + flatbuffers::Offset state_response = 0, + const std::vector *hash = nullptr) { + auto hash__ = hash ? _fbb.CreateVector(*hash) : 0; + return fbschema::p2pmsg::CreateState_Response_Message( _fbb, - previous__, - current__, - difference, - patch__); + state_response_type, + state_response, + hash__); +} + +struct Fs_Entry_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PATH = 4, + VT_ENTRIES = 6 + }; + const flatbuffers::String *path() const { + return GetPointer(VT_PATH); + } + flatbuffers::String *mutable_path() { + return GetPointer(VT_PATH); + } + const flatbuffers::Vector> *entries() const { + return GetPointer> *>(VT_ENTRIES); + } + flatbuffers::Vector> *mutable_entries() { + return GetPointer> *>(VT_ENTRIES); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PATH) && + verifier.VerifyString(path()) && + VerifyOffset(verifier, VT_ENTRIES) && + verifier.VerifyVector(entries()) && + verifier.VerifyVectorOfTables(entries()) && + verifier.EndTable(); + } +}; + +struct Fs_Entry_ResponseBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_path(flatbuffers::Offset path) { + fbb_.AddOffset(Fs_Entry_Response::VT_PATH, path); + } + void add_entries(flatbuffers::Offset>> entries) { + fbb_.AddOffset(Fs_Entry_Response::VT_ENTRIES, entries); + } + explicit Fs_Entry_ResponseBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + Fs_Entry_ResponseBuilder &operator=(const Fs_Entry_ResponseBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateFs_Entry_Response( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset path = 0, + flatbuffers::Offset>> entries = 0) { + Fs_Entry_ResponseBuilder builder_(_fbb); + builder_.add_entries(entries); + builder_.add_path(path); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateFs_Entry_ResponseDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *path = nullptr, + const std::vector> *entries = nullptr) { + auto path__ = path ? _fbb.CreateString(path) : 0; + auto entries__ = entries ? _fbb.CreateVector>(*entries) : 0; + return fbschema::p2pmsg::CreateFs_Entry_Response( + _fbb, + path__, + entries__); +} + +struct File_HashMap_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PATH = 4, + VT_FILE_LENGTH = 6, + VT_HASH_MAP = 8 + }; + const flatbuffers::String *path() const { + return GetPointer(VT_PATH); + } + flatbuffers::String *mutable_path() { + return GetPointer(VT_PATH); + } + uint64_t file_length() const { + return GetField(VT_FILE_LENGTH, 0); + } + bool mutate_file_length(uint64_t _file_length) { + return SetField(VT_FILE_LENGTH, _file_length, 0); + } + const flatbuffers::Vector *hash_map() const { + return GetPointer *>(VT_HASH_MAP); + } + flatbuffers::Vector *mutable_hash_map() { + return GetPointer *>(VT_HASH_MAP); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PATH) && + verifier.VerifyString(path()) && + VerifyField(verifier, VT_FILE_LENGTH) && + VerifyOffset(verifier, VT_HASH_MAP) && + verifier.VerifyVector(hash_map()) && + verifier.EndTable(); + } +}; + +struct File_HashMap_ResponseBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_path(flatbuffers::Offset path) { + fbb_.AddOffset(File_HashMap_Response::VT_PATH, path); + } + void add_file_length(uint64_t file_length) { + fbb_.AddElement(File_HashMap_Response::VT_FILE_LENGTH, file_length, 0); + } + void add_hash_map(flatbuffers::Offset> hash_map) { + fbb_.AddOffset(File_HashMap_Response::VT_HASH_MAP, hash_map); + } + explicit File_HashMap_ResponseBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + File_HashMap_ResponseBuilder &operator=(const File_HashMap_ResponseBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateFile_HashMap_Response( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset path = 0, + uint64_t file_length = 0, + flatbuffers::Offset> hash_map = 0) { + File_HashMap_ResponseBuilder builder_(_fbb); + builder_.add_file_length(file_length); + builder_.add_hash_map(hash_map); + builder_.add_path(path); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateFile_HashMap_ResponseDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *path = nullptr, + uint64_t file_length = 0, + const std::vector *hash_map = nullptr) { + auto path__ = path ? _fbb.CreateString(path) : 0; + auto hash_map__ = hash_map ? _fbb.CreateVector(*hash_map) : 0; + return fbschema::p2pmsg::CreateFile_HashMap_Response( + _fbb, + path__, + file_length, + hash_map__); +} + +struct Block_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PATH = 4, + VT_BLOCK_ID = 6, + VT_DATA = 8 + }; + const flatbuffers::String *path() const { + return GetPointer(VT_PATH); + } + flatbuffers::String *mutable_path() { + return GetPointer(VT_PATH); + } + uint32_t block_id() const { + return GetField(VT_BLOCK_ID, 0); + } + bool mutate_block_id(uint32_t _block_id) { + return SetField(VT_BLOCK_ID, _block_id, 0); + } + const flatbuffers::Vector *data() const { + return GetPointer *>(VT_DATA); + } + flatbuffers::Vector *mutable_data() { + return GetPointer *>(VT_DATA); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PATH) && + verifier.VerifyString(path()) && + VerifyField(verifier, VT_BLOCK_ID) && + VerifyOffset(verifier, VT_DATA) && + verifier.VerifyVector(data()) && + verifier.EndTable(); + } +}; + +struct Block_ResponseBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_path(flatbuffers::Offset path) { + fbb_.AddOffset(Block_Response::VT_PATH, path); + } + void add_block_id(uint32_t block_id) { + fbb_.AddElement(Block_Response::VT_BLOCK_ID, block_id, 0); + } + void add_data(flatbuffers::Offset> data) { + fbb_.AddOffset(Block_Response::VT_DATA, data); + } + explicit Block_ResponseBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + Block_ResponseBuilder &operator=(const Block_ResponseBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateBlock_Response( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset path = 0, + uint32_t block_id = 0, + flatbuffers::Offset> data = 0) { + Block_ResponseBuilder builder_(_fbb); + builder_.add_data(data); + builder_.add_block_id(block_id); + builder_.add_path(path); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateBlock_ResponseDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *path = nullptr, + uint32_t block_id = 0, + const std::vector *data = nullptr) { + auto path__ = path ? _fbb.CreateString(path) : 0; + auto data__ = data ? _fbb.CreateVector(*data) : 0; + return fbschema::p2pmsg::CreateBlock_Response( + _fbb, + path__, + block_id, + data__); +} + +struct State_FS_Hash_Entry FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_PATH = 4, + VT_IS_FILE = 6, + VT_HASH = 8 + }; + const flatbuffers::String *path() const { + return GetPointer(VT_PATH); + } + flatbuffers::String *mutable_path() { + return GetPointer(VT_PATH); + } + bool is_file() const { + return GetField(VT_IS_FILE, 0) != 0; + } + bool mutate_is_file(bool _is_file) { + return SetField(VT_IS_FILE, static_cast(_is_file), 0); + } + const flatbuffers::Vector *hash() const { + return GetPointer *>(VT_HASH); + } + flatbuffers::Vector *mutable_hash() { + return GetPointer *>(VT_HASH); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffset(verifier, VT_PATH) && + verifier.VerifyString(path()) && + VerifyField(verifier, VT_IS_FILE) && + VerifyOffset(verifier, VT_HASH) && + verifier.VerifyVector(hash()) && + verifier.EndTable(); + } +}; + +struct State_FS_Hash_EntryBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_path(flatbuffers::Offset path) { + fbb_.AddOffset(State_FS_Hash_Entry::VT_PATH, path); + } + void add_is_file(bool is_file) { + fbb_.AddElement(State_FS_Hash_Entry::VT_IS_FILE, static_cast(is_file), 0); + } + void add_hash(flatbuffers::Offset> hash) { + fbb_.AddOffset(State_FS_Hash_Entry::VT_HASH, hash); + } + explicit State_FS_Hash_EntryBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + State_FS_Hash_EntryBuilder &operator=(const State_FS_Hash_EntryBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + return o; + } +}; + +inline flatbuffers::Offset CreateState_FS_Hash_Entry( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset path = 0, + bool is_file = false, + flatbuffers::Offset> hash = 0) { + State_FS_Hash_EntryBuilder builder_(_fbb); + builder_.add_hash(hash); + builder_.add_path(path); + builder_.add_is_file(is_file); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateState_FS_Hash_EntryDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *path = nullptr, + bool is_file = false, + const std::vector *hash = nullptr) { + auto path__ = path ? _fbb.CreateString(path) : 0; + auto hash__ = hash ? _fbb.CreateVector(*hash) : 0; + return fbschema::p2pmsg::CreateState_FS_Hash_Entry( + _fbb, + path__, + is_file, + hash__); } inline bool VerifyMessage(flatbuffers::Verifier &verifier, const void *obj, Message type) { @@ -1091,6 +1538,14 @@ inline bool VerifyMessage(flatbuffers::Verifier &verifier, const void *obj, Mess auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case Message_State_Request_Message: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case Message_State_Response_Message: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } case Message_History_Request_Message: { auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); @@ -1115,6 +1570,39 @@ inline bool VerifyMessageVector(flatbuffers::Verifier &verifier, const flatbuffe return true; } +inline bool VerifyState_Response(flatbuffers::Verifier &verifier, const void *obj, State_Response type) { + switch (type) { + case State_Response_NONE: { + return true; + } + case State_Response_File_HashMap_Response: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case State_Response_Block_Response: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + case State_Response_Fs_Entry_Response: { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } + default: return false; + } +} + +inline bool VerifyState_ResponseVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector> *values, const flatbuffers::Vector *types) { + if (!values || !types) return !values && !types; + if (values->size() != types->size()) return false; + for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) { + if (!VerifyState_Response( + verifier, values->Get(i), types->GetEnum(i))) { + return false; + } + } + return true; +} + inline const fbschema::p2pmsg::Content *GetContent(const void *buf) { return flatbuffers::GetRoot(buf); } diff --git a/src/fbschema/p2pmsg_helpers.cpp b/src/fbschema/p2pmsg_helpers.cpp index 4d232def..1d81cd21 100644 --- a/src/fbschema/p2pmsg_helpers.cpp +++ b/src/fbschema/p2pmsg_helpers.cpp @@ -166,7 +166,7 @@ const p2p::history_response create_history_response_from_msg(const History_Respo if (msg.hist_ledgers()) hr.hist_ledgers = flatbuf_historyledgermap_to_historyledgermap(msg.hist_ledgers()); - if (msg.error()) + if (msg.error()) hr.error = (p2p::LEDGER_RESPONSE_ERROR)msg.error(); return hr; @@ -174,7 +174,7 @@ const p2p::history_response create_history_response_from_msg(const History_Respo /** * Creates a proposal stuct from the given proposal message. - * @param The Flatbuffer poporal received from the peer. + * @param The Flatbuffer poposal received from the peer. * @return A proposal struct representing the message. */ const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector *pubkey, const uint64_t timestamp, const flatbuffers::Vector *lcl) @@ -186,6 +186,7 @@ const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const p.time = msg.time(); p.stage = msg.stage(); p.lcl = flatbuff_bytes_to_sv(lcl); + p.curr_hash_state = flatbuff_bytes_to_sv(msg.curr_state_hash()); if (msg.users()) p.users = flatbuf_bytearrayvector_to_stringlist(msg.users()); @@ -217,6 +218,38 @@ const p2p::history_request create_history_request_from_msg(const History_Request return hr; } +/** + * Creates a state request struct from the given state request message. + * @param msg Flatbuffer State request message received from the peer. + * @return A State request struct representing the message. + */ +const p2p::state_request create_state_request_from_msg(const State_Request_Message &msg) +{ + p2p::state_request sr; + + sr.block_id = msg.block_id(); + sr.is_file = msg.is_file(); + sr.parent_path = flatbuff_str_to_sv(msg.parent_path()); + sr.expected_hash = flatbuff_bytes_to_hash(msg.expected_hash()); + + return sr; +} + +/** + * Creates a block response struct from the given block response message. + * @param msg Flatbuffer block response message received from the peer. + * @return A Block response struct representing the message. + */ +const p2p::block_response create_block_response_from_msg(const Block_Response &msg) +{ + p2p::block_response br; + + br.path = flatbuff_str_to_sv(msg.path()); + br.block_id = msg.block_id(); + br.data = flatbuff_bytes_to_sv(msg.data()); + return br; +} + //---Message creation helpers---// void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup) @@ -253,7 +286,8 @@ void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, p.time, stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), - stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs)); + stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs), + sv_to_flatbuff_bytes(builder, p.curr_hash_state)); const flatbuffers::Offset message = CreateContent(builder, Message_Proposal_Message, proposal.Union()); builder.Finish(message); // Finished building message content to get serialised content. @@ -332,6 +366,144 @@ void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_ create_containermsg_from_content(container_builder, builder, nullptr, true); } +/** + * Create state request message from the given state request struct. + * @param container_builder Flatbuffer builder for the container message. + * @param sr The state request struct to be placed in the container message. + */ +void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::state_request &hr, std::string_view lcl) +{ + flatbuffers::FlatBufferBuilder builder(1024); + + flatbuffers::Offset srmsg = + CreateState_Request_Message( + builder, + sv_to_flatbuff_str(builder, hr.parent_path), + hr.is_file, + hr.block_id, + hash_to_flatbuff_bytes(builder, hr.expected_hash)); + + flatbuffers::Offset message = CreateContent(builder, Message_State_Request_Message, srmsg.Union()); + builder.Finish(message); // Finished building message content to get serialised content. + + // Now that we have built the content message, + // we need to sign it and place it inside a container message. + create_containermsg_from_content(container_builder, builder, lcl, true); +} + +/** + * Create content response message from the given content response. + * @param container_builder Flatbuffer builder for the container message. + * @param path The path of the directory. + * @param fs_entries File or directory entries in the given parent path. + * @param expected_hash The exptected hash of the requested path. + * @param lcl Lcl to be include in the container msg. + */ +void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, std::unordered_map &fs_entries, hasher::B2H expected_hash, std::string_view lcl) +{ + flatbuffers::FlatBufferBuilder builder(1024); + + const flatbuffers::Offset resp = + CreateFs_Entry_Response( + builder, + sv_to_flatbuff_str(builder, path), + statefshashentry_to_flatbuff_statefshashentry(builder, fs_entries)); + + const flatbuffers::Offset st_resp = CreateState_Response_Message( + builder, State_Response_Fs_Entry_Response, + resp.Union(), + hash_to_flatbuff_bytes(builder, expected_hash)); + + flatbuffers::Offset message = CreateContent(builder, Message_State_Response_Message, st_resp.Union()); + builder.Finish(message); // Finished building message content to get serialised content. + + // Now that we have built the content message, + // we need to sign it and place it inside a container message. + create_containermsg_from_content(container_builder, builder, lcl, true); +} + +/** + * Create content response message from the given content response. + * @param container_builder Flatbuffer builder for the container message. + * @param path The path of the directory. + * @param hashmap Hashmap of the file + * @param lcl Lcl to be include in the container msg. + */ +void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hasher::B2H expected_hash, std::string_view lcl) +{ + // todo:get a average propsal message size and allocate content builder based on that. + flatbuffers::FlatBufferBuilder builder(1024); + + std::string_view hashmap_sv(reinterpret_cast(hashmap.data()), hashmap.size()); + + const flatbuffers::Offset resp = + CreateFile_HashMap_Response( + builder, + sv_to_flatbuff_str(builder, path), + file_length, + sv_to_flatbuff_bytes(builder, hashmap_sv)); + + const flatbuffers::Offset st_resp = CreateState_Response_Message( + builder, + State_Response_File_HashMap_Response, + resp.Union(), + hash_to_flatbuff_bytes(builder, expected_hash)); + + flatbuffers::Offset message = CreateContent(builder, Message_State_Response_Message, st_resp.Union()); + builder.Finish(message); // Finished building message content to get serialised content. + + // Now that we have built the content message, + // we need to sign it and place it inside a container message. + create_containermsg_from_content(container_builder, builder, lcl, true); +} + +/** + * Create content response message from the given content response. + * @param container_builder Flatbuffer builder for the container message. + * @param block_resp Block response struct to place in the message + * @param lcl Lcl to be include in the container message. + */ +void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl) +{ + // todo:get a average propsal message size and allocate content builder based on that. + flatbuffers::FlatBufferBuilder builder(1024); + + const flatbuffers::Offset resp = + CreateBlock_Response( + builder, + sv_to_flatbuff_str(builder, block_resp.path), + block_resp.block_id, + sv_to_flatbuff_bytes(builder, block_resp.data)); + + const flatbuffers::Offset st_resp = CreateState_Response_Message( + builder, + State_Response_Block_Response, + resp.Union(), + hash_to_flatbuff_bytes(builder, block_resp.hash)); + + flatbuffers::Offset message = CreateContent(builder, Message_State_Response_Message, st_resp.Union()); + builder.Finish(message); // Finished building message content to get serialised content. + + // Now that we have built the content message, + // we need to sign it and place it inside a container message. + create_containermsg_from_content(container_builder, builder, lcl, true); +} + +void create_msg_from_state_error_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl) +{ + // todo:get a average propsal message size and allocate content builder based on that. + flatbuffers::FlatBufferBuilder builder(1024); + + const flatbuffers::Offset st_resp = CreateState_Response_Message(builder, State_Response_NONE, 0, true); + + flatbuffers::Offset message = CreateContent(builder, Message_State_Response_Message, st_resp.Union()); + builder.Finish(message); // Finished building message content to get serialised content. + + // Now that we have built the content message, + // we need to sign it and place it inside a container message. + create_containermsg_from_content(container_builder, builder, lcl, true); +} + /** * Creates a Flatbuffer container message from the given Content message. * @param container_builder The Flatbuffer builder to which the final container message should be written to. @@ -457,6 +629,7 @@ historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &bui { flatbuffers::Offset history_ledger = CreateHistoryLedger( builder, + sv_to_flatbuff_bytes(builder, ledger.state), sv_to_flatbuff_bytes(builder, ledger.lcl), builder.CreateVector(ledger.raw_ledger)); @@ -468,4 +641,35 @@ historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &bui return builder.CreateVector(fbvec); } +void flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, const flatbuffers::Vector> *fhashes) +{ + + for (const State_FS_Hash_Entry *f_hash : *fhashes) + { + p2p::state_fs_hash_entry h; + + h.is_file = f_hash->is_file(); + h.hash = flatbuff_bytes_to_hash(f_hash->hash()); + fs_entries.emplace(flatbuff_str_to_sv(f_hash->path()), std::move(h)); + } +} + +flatbuffers::Offset>> +statefshashentry_to_flatbuff_statefshashentry(flatbuffers::FlatBufferBuilder &builder, std::unordered_map &fs_entries) +{ + std::vector> fbvec; + fbvec.reserve(fs_entries.size()); + for (auto const &[path, fs_entry] : fs_entries) + { + flatbuffers::Offset state_fs_entry = CreateState_FS_Hash_Entry( + builder, + sv_to_flatbuff_str(builder, path), + fs_entry.is_file, + hash_to_flatbuff_bytes(builder, fs_entry.hash)); + + fbvec.push_back(state_fs_entry); + } + return builder.CreateVector(fbvec); +} + } // namespace fbschema::p2pmsg \ No newline at end of file diff --git a/src/fbschema/p2pmsg_helpers.hpp b/src/fbschema/p2pmsg_helpers.hpp index 88735c92..afc1eea0 100644 --- a/src/fbschema/p2pmsg_helpers.hpp +++ b/src/fbschema/p2pmsg_helpers.hpp @@ -6,6 +6,7 @@ #include "p2pmsg_container_generated.h" #include "p2pmsg_content_generated.h" #include "../p2p/p2p.hpp" +#include "../statefs/hasher.hpp" namespace fbschema::p2pmsg { @@ -29,6 +30,10 @@ const p2p::history_request create_history_request_from_msg(const History_Request const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg); +const p2p::state_request create_state_request_from_msg(const State_Request_Message &msg); + +const p2p::block_response create_block_response_from_msg(const Block_Response &msg); + //---Message creation helpers---// void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup); @@ -41,6 +46,15 @@ void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_ void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const p2p::npl_message &npl, std::string_view lcl); +void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::state_request &hr, std::string_view lcl); + +void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, +std::unordered_map &fs_entries, hasher::B2H expected_hash, std::string_view lcl); + +void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hasher::B2H expected_hash, std::string_view lcl); + +void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl); + void create_containermsg_from_content( flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign); @@ -60,6 +74,17 @@ flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector>> historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map); +void +flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, +const flatbuffers::Vector> *fhashes); + +void statefilehash_to_flatbuf_statefilehash(flatbuffers::FlatBufferBuilder &builder, std::vector> &list, + std::string_view full_path, bool is_file, std::string_view hash); + + +flatbuffers::Offset>> +statefshashentry_to_flatbuff_statefshashentry(flatbuffers::FlatBufferBuilder &builder, std::unordered_map &fs_entries); + } // namespace fbschema::p2pmsg #endif \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index adab7a3c..eee84aa7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -79,7 +79,7 @@ void signal_handler(int signum) */ void boost::throw_exception(std::exception const &e) { - LOG_ERR << "Boost error:" << e.what(); + std::cerr << "Boost error:" << e.what() << "\n"; exit(1); } @@ -97,16 +97,16 @@ void std_terminate() noexcept } catch (std::exception &ex) { - LOG_ERR << "std error: " << ex.what(); + std::cerr << "std error: " << ex.what() << "\n"; } catch (...) { - LOG_ERR << "std error: Terminated due to unknown exception"; + std::cerr << "std error: Terminated due to unknown exception" << "\n"; } } else { - LOG_ERR << "std error: Terminated due to unknown reason"; + std::cerr << "std error: Terminated due to unknown reason" << "\n"; } exit(1); @@ -170,11 +170,11 @@ int main(int argc, char **argv) LOG_INFO << "Operating mode: " << (conf::cfg.mode == conf::OPERATING_MODE::OBSERVING ? "Observing" : "Proposing"); + statefs::init(conf::ctx.statehistdir); + if (p2p::init() != 0 || usr::init() != 0 || cons::init() != 0) return -1; - statefs::init(conf::ctx.statehistdir); - // After initializing primary subsystems, register the SIGINT handler. signal(SIGINT, signal_handler); diff --git a/src/p2p/p2p.cpp b/src/p2p/p2p.cpp index fd939ae7..4eb2ea8a 100644 --- a/src/p2p/p2p.cpp +++ b/src/p2p/p2p.cpp @@ -103,6 +103,9 @@ void broadcast_message(const peer_outbound_message msg, bool send_to_self) */ void send_message_to_random_peer(peer_outbound_message msg) { + //Send while locking the peer_connections. + std::lock_guard lock(p2p::ctx.peer_connections_mutex); + size_t connected_peers = ctx.peer_connections.size(); if (connected_peers == 0) { @@ -115,19 +118,20 @@ void send_message_to_random_peer(peer_outbound_message msg) return; } - //Send while locking the peer_connections. - std::lock_guard lock(p2p::ctx.peer_connections_mutex); - - // Initialize random number generator with current timestamp. - int random_peer_index = (rand() % connected_peers); // select a random peer index. - auto it = ctx.peer_connections.begin(); - std::advance(it, random_peer_index); //move iterator to point to random selected peer. - - //send message to selecte peer. - auto session = it->second; - if (!session->is_self) + while (true) { - session->send(msg); + // Initialize random number generator with current timestamp. + int random_peer_index = (rand() % connected_peers); // select a random peer index. + auto it = ctx.peer_connections.begin(); + std::advance(it, random_peer_index); //move iterator to point to random selected peer. + + //send message to selecte peer. + auto session = it->second; + if (!session->is_self) + { + session->send(msg); + break; + } } } diff --git a/src/p2p/p2p.hpp b/src/p2p/p2p.hpp index 13c39ea4..f9293b46 100644 --- a/src/p2p/p2p.hpp +++ b/src/p2p/p2p.hpp @@ -5,6 +5,7 @@ #include "../sock/socket_session.hpp" #include "../usr/user_input.hpp" #include "peer_session_handler.hpp" +#include "../statefs/hasher.hpp" namespace p2p { @@ -16,6 +17,7 @@ struct proposal uint64_t time; uint8_t stage; std::string lcl; + std::string curr_hash_state; std::set users; std::set hash_inputs; std::set hash_outputs; @@ -34,6 +36,7 @@ struct history_request struct history_ledger { + std::string state; std::string lcl; std::vector raw_ledger; }; @@ -45,30 +48,53 @@ enum LEDGER_RESPONSE_ERROR REQ_LEDGER_NOT_FOUND = 2 }; - struct history_response { - std::map hist_ledgers; + std::map hist_ledgers; LEDGER_RESPONSE_ERROR error; - }; - + struct npl_message { std::string data; }; +struct state_request +{ + std::string parent_path; + bool is_file; + int32_t block_id; + hasher::B2H expected_hash; +}; + +struct state_fs_hash_entry +{ + bool is_file; + hasher::B2H hash; +}; + +struct block_response +{ + std::string path; + uint32_t block_id; + std::string_view data; + hasher::B2H hash; +}; + struct message_collection { - std::list proposals; - std::mutex proposals_mutex; // Mutex for proposals access race conditions. - + std::list proposals; + std::mutex proposals_mutex; // Mutex for proposals access race conditions. + std::list nonunl_proposals; - std::mutex nonunl_proposals_mutex; // Mutex for non-unl proposals access race conditions. + std::mutex nonunl_proposals_mutex; // Mutex for non-unl proposals access race conditions. // NPL messages are stored as string list because we are feeding the npl messages as it is (byte array) to the contract. - std::list npl_messages; - std::mutex npl_messages_mutex; // Mutex for npl_messages access race conditions. + std::list npl_messages; + std::mutex npl_messages_mutex; // Mutex for npl_messages access race conditions. + + std::list state_response; + std::mutex state_response_mutex; // Mutex for state response access race conditions. }; struct connected_context diff --git a/src/p2p/peer_session_handler.cpp b/src/p2p/peer_session_handler.cpp index 3927fe2d..db915603 100644 --- a/src/p2p/peer_session_handler.cpp +++ b/src/p2p/peer_session_handler.cpp @@ -12,6 +12,8 @@ #include "p2p.hpp" #include "peer_session_handler.hpp" #include "../cons/ledger_handler.hpp" +#include "../cons/state_handler.hpp" +#include "../cons/cons.hpp" namespace p2pmsg = fbschema::p2pmsg; @@ -110,10 +112,29 @@ void peer_session_handler::on_message(sock::socket_session(container_buf_ptr), container_buf_size); ctx.collected_msgs.npl_messages.push_back(std::move(npl_message)); } + else if (content_message_type == p2pmsg::Message_State_Request_Message) + { + if (p2pmsg::validate_container_trust(container) != 0) + { + LOG_DBG << "State request message rejected due to trust failure."; + return; + } + + const p2p::state_request sr = p2pmsg::create_state_request_from_msg(*content->message_as_State_Request_Message()); + p2p::peer_outbound_message msg(std::make_unique(1024)); + + if (cons::create_state_response(msg, sr) == 0) + session->send(std::move(msg)); + } + else if (content_message_type == p2pmsg::Message_State_Response_Message) + { + LOG_INFO << "Received State Response Message\n"; + std::lock_guard lock(ctx.collected_msgs.state_response_mutex); // Insert state_response with lock. + std::string response(reinterpret_cast(content_ptr), content_size); + ctx.collected_msgs.state_response.push_back(std::move(response)); + } else if (content_message_type == p2pmsg::Message_History_Request_Message) //message is a lcl history request message { - LOG_DBG << "Received history request message type from peer."; - const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message()); //first check node has the required lcl available. -> if so send lcl history accordingly. bool req_lcl_avail = cons::check_required_lcl_availability(hr); @@ -125,8 +146,6 @@ void peer_session_handler::on_message(sock::socket_sessionmessage_as_History_Response_Message())); } diff --git a/src/pchheader.hpp b/src/pchheader.hpp index 7d8cf8ae..63165079 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include diff --git a/src/proc/proc.cpp b/src/proc/proc.cpp index 907ab697..fc9a2d19 100644 --- a/src/proc/proc.cpp +++ b/src/proc/proc.cpp @@ -8,6 +8,7 @@ #include "../statefs/state_common.hpp" #include "../statefs/hashtree_builder.hpp" #include "proc.hpp" +#include "../cons/cons.hpp" namespace proc { @@ -40,6 +41,8 @@ pid_t contract_pid; // Holds the state monitor process id (if currently executing). pid_t statemon_pid; +const char *FINDMNT_COMMAND = "findmnt --noheadings "; + /** * Executes the contract process and passes the specified arguments. * @return 0 on successful process creation. -1 on failure or contract process is already running. @@ -144,7 +147,22 @@ int start_state_monitor() { // HP process. statemon_pid = pid; - return 0; + + // Give enough time for the state monitor to start. + // We wait until Fuse filesystem is mounted for max number of retries. + uint16_t retry_count = 0; + std::string findmnt_command = FINDMNT_COMMAND + conf::ctx.statedir; + while (retry_count < 50) + { + util::sleep(10); + int ret = system(findmnt_command.c_str()); + if (WEXITSTATUS(ret) == 0) // Success. Fuse fs has been mounted. + return 0; + retry_count++; + } + + // We waited enough time for Fuse fs to be mounted but no luck. + return -1; } else if (pid == 0) { @@ -186,11 +204,14 @@ int stop_state_monitor() LOG_ERR << "State monitor process exited with non-normal status code: " << presult; // Update the hash tree. - hasher::B2H statehash = {0, 0, 0, 0}; + hasher::B2H statehash = hasher::B2H_empty; statefs::hashtree_builder htreebuilder(statefs::get_statedir_context()); if (htreebuilder.generate(statehash) != 0) return -1; + std::string root_hash(reinterpret_cast(&statehash), hasher::HASH_SIZE); + root_hash.swap(cons::ctx.curr_hash_state); + LOG_DBG << "State hash: " << std::hex << statehash << std::dec; return 0; diff --git a/src/sock/socket_message.cpp b/src/sock/socket_message.cpp index dea51bb8..aed3922a 100644 --- a/src/sock/socket_message.cpp +++ b/src/sock/socket_message.cpp @@ -5,9 +5,9 @@ namespace usr { -user_outbound_message::user_outbound_message(std::string &&_msg) +user_outbound_message::user_outbound_message(std::string &&msg) { - msg = std::move(_msg); + this->msg = std::move(msg); } // Returns the buffer that should be written to the socket. @@ -22,9 +22,9 @@ namespace p2p { peer_outbound_message::peer_outbound_message( - std::shared_ptr _fbbuilder_ptr) + std::shared_ptr fbbuilder_ptr) { - fbbuilder_ptr = _fbbuilder_ptr; + this->fbbuilder_ptr = fbbuilder_ptr; } // Returns a reference to the flatbuffer builder object. @@ -37,8 +37,8 @@ flatbuffers::FlatBufferBuilder &peer_outbound_message::builder() std::string_view peer_outbound_message::buffer() { return std::string_view( - reinterpret_cast((*fbbuilder_ptr).GetBufferPointer()), - (*fbbuilder_ptr).GetSize()); + reinterpret_cast(fbbuilder_ptr->GetBufferPointer()), + fbbuilder_ptr->GetSize()); } } \ No newline at end of file diff --git a/src/sock/socket_session.cpp b/src/sock/socket_session.cpp index 49a83deb..392ca8f4 100644 --- a/src/sock/socket_session.cpp +++ b/src/sock/socket_session.cpp @@ -72,7 +72,6 @@ void socket_session::increment_metric(const SESSION_THRESHOLDS threshold_type LOG_INFO << "Session " << this->uniqueid << " threshold exceeded. (type:" << threshold_type << " limit:" << t.threshold_limit << ")"; corebill::report_violation(this->address); - } else if (elapsed_time > t.intervalms) { @@ -119,7 +118,7 @@ void socket_session::run(const std::string &&address, const std::string &&por this->uniqueid.append(address).append(":").append(port); // This indicates the connection is a self connection (node connects to the same node through server port) - if(address == "0.0.0.0") + if (address == "0.0.0.0") this->is_self = true; // Set the timeout. diff --git a/src/statefs/hasher.cpp b/src/statefs/hasher.cpp index 18bef6b5..58134082 100644 --- a/src/statefs/hasher.cpp +++ b/src/statefs/hasher.cpp @@ -1,25 +1,38 @@ #include "hasher.hpp" +/** + * Contains hashing functions and helpers used to manipulate block hashes used in state management. + * This could also be used throughout rest of the application as well. However for now we are only + * using this for state management code base only. + * + * Based on https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp + */ namespace hasher { -// provide some helper functions for working with 32 byte hash type -bool operator==(const B2H &lhs, const B2H &rhs) +// Represents empty/default B2H hash value. +B2H B2H_empty = hasher::B2H_empty; + +/** + * Helper functions for working with 32 byte hash type B2H. + */ + +bool B2H::operator==(const B2H rhs) const { - return lhs.data[0] == rhs.data[0] && lhs.data[1] == rhs.data[1] && lhs.data[2] == rhs.data[2] && lhs.data[3] == rhs.data[3]; + return this->data[0] == rhs.data[0] && this->data[1] == rhs.data[1] && this->data[2] == rhs.data[2] && this->data[3] == rhs.data[3]; } -bool operator!=(const B2H &lhs, const B2H &rhs) +bool B2H::operator!=(const B2H rhs) const { - return lhs.data[0] != rhs.data[0] || lhs.data[1] != rhs.data[1] || lhs.data[2] != rhs.data[2] || lhs.data[3] != rhs.data[3]; + return this->data[0] != rhs.data[0] || this->data[1] != rhs.data[1] || this->data[2] != rhs.data[2] || this->data[3] != rhs.data[3]; } -void operator^=(B2H &lhs, const B2H &rhs) +void B2H::operator^=(const B2H rhs) { - lhs.data[0] ^= rhs.data[0]; - lhs.data[1] ^= rhs.data[1]; - lhs.data[2] ^= rhs.data[2]; - lhs.data[3] ^= rhs.data[3]; + this->data[0] ^= rhs.data[0]; + this->data[1] ^= rhs.data[1]; + this->data[2] ^= rhs.data[2]; + this->data[3] ^= rhs.data[3]; } std::ostream &operator<<(std::ostream &output, const B2H &h) @@ -34,8 +47,9 @@ std::stringstream &operator<<(std::stringstream &output, const B2H &h) return output; } -// the actual hash function, note that the B2H datatype is always passed by value being only 4 quadwords -B2H hash(const void *buf1, size_t buf1len, const void *buf2, size_t buf2len) +// The actual hash function, note that the B2H datatype is always passed by value being only 4 quadwords. +// This function accepts two buffers to hash together in order to support common use case in state handling. +B2H hash(const void *buf1, const size_t buf1len, const void *buf2, const size_t buf2len) { crypto_generichash_blake2b_state state; crypto_generichash_blake2b_init(&state, NULL, 0, HASH_SIZE); @@ -52,4 +66,17 @@ B2H hash(const void *buf1, size_t buf1len, const void *buf2, size_t buf2len) return ret; } +// Helper class to support std::map/std::unordered_map custom hashing function. +// This is needed to use B2H as the std map container key. +size_t B2H_std_key_hasher::operator()(const hasher::B2H h) const +{ + // Compute individual hash values. http://stackoverflow.com/a/1646913/126995 + size_t res = 17; + res = res * 31 + std::hash()(h.data[0]); + res = res * 31 + std::hash()(h.data[1]); + res = res * 31 + std::hash()(h.data[2]); + res = res * 31 + std::hash()(h.data[3]); + return res; +} + } // namespace hasher \ No newline at end of file diff --git a/src/statefs/hasher.hpp b/src/statefs/hasher.hpp index 04869b48..26006cf1 100644 --- a/src/statefs/hasher.hpp +++ b/src/statefs/hasher.hpp @@ -6,21 +6,34 @@ namespace hasher { +// Hash length (32 bytes) constexpr size_t HASH_SIZE = crypto_generichash_blake2b_BYTES; -struct B2H // blake2b hash is 32 bytes which we store as 4 quad words +// blake2b hash is 32 bytes which we store as 4 quad words +// Originally from https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp +struct B2H { uint64_t data[4]; + + bool operator==(const B2H rhs) const; + bool operator!=(const B2H rhs) const; + void operator^=(const B2H rhs); }; -// provide some helper functions for working with 32 byte hash type -bool operator==(const B2H &lhs, const B2H &rhs); -bool operator!=(const B2H &lhs, const B2H &rhs); -void operator^=(B2H &lhs, const B2H &rhs); +extern B2H B2H_empty; + std::ostream &operator<<(std::ostream &output, const B2H &h); std::stringstream &operator<<(std::stringstream &output, const B2H &h); -B2H hash(const void *buf1, size_t buf1len, const void *buf2, size_t buf2len); +B2H hash(const void *buf1, const size_t buf1len, const void *buf2, const size_t buf2len); + +// Helper class to support std::map/std::unordered_map custom hashing function. +// This is needed to use B2H as the std map container key. +class B2H_std_key_hasher +{ +public: + size_t operator()(const hasher::B2H h) const; +}; } // namespace hasher diff --git a/src/statefs/hashmap_builder.cpp b/src/statefs/hashmap_builder.cpp index f2f0b344..eec2e017 100644 --- a/src/statefs/hashmap_builder.cpp +++ b/src/statefs/hashmap_builder.cpp @@ -11,17 +11,18 @@ hashmap_builder::hashmap_builder(const statedir_context &ctx) : ctx(ctx) { } -int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath) +int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath, const std::string filerelpath, const std::map &changedblocks) { // We attempt to avoid a full rebuild of the block hash map file when possible. // For this optimisation, both the block hash map (.bhmap) file and the - // delta block index (.bindex) file must exist. + // delta block index must exist. + + // Block index may be provided as an argument. If it is empty we attempt to read from the + // .bindex file from the state checkpoint delta. // If the block index exists, we generate/update the hashmap file with the aid of that. // Block index file contains the updated blockids. If not, we simply rehash all the blocks. - std::string relpath = get_relpath(filepath, ctx.datadir); - // Open the actual data file and calculate the block count. int orifd = open(filepath.data(), O_RDONLY); if (orifd == -1) @@ -35,31 +36,41 @@ int hashmap_builder::generate_hashmap_forfile(hasher::B2H &parentdirhash, const // Attempt to read the existing block hash map file. std::string bhmapfile; std::vector bhmapdata; - if (read_blockhashmap(bhmapdata, bhmapfile, relpath) == -1) + if (read_blockhashmap(bhmapdata, bhmapfile, filerelpath) == -1) return -1; - hasher::B2H oldfilehash = {0, 0, 0, 0}; + hasher::B2H oldfilehash = hasher::B2H_empty; if (!bhmapdata.empty()) memcpy(&oldfilehash, bhmapdata.data(), hasher::HASH_SIZE); - // Attempt to read the delta block index file. - std::map bindex; - uint32_t original_blockcount; - if (get_blockindex(bindex, original_blockcount, relpath) == -1) - return -1; - // Array to contain the updated block hashes. Slot 0 is for the root hash. // Allocating hash array on the heap to avoid filling limited stack space. std::unique_ptr hashes = std::make_unique(1 + blockcount); const size_t hashes_size = (1 + blockcount) * hasher::HASH_SIZE; - if (update_hashes(hashes.get(), hashes_size, relpath, orifd, blockcount, original_blockcount, bindex, bhmapdata) == -1) - return -1; + if (changedblocks.empty()) + { + // Attempt to read the delta block index file. + std::map bindex; + uint32_t original_blockcount; + if (get_blockindex(bindex, original_blockcount, filerelpath) == -1) + return -1; + + if (update_hashes_with_backup_blockhints(hashes.get(), hashes_size, filerelpath, orifd, blockcount, original_blockcount, bindex, bhmapdata) == -1) + return -1; + } + else + { + if (update_hashes_with_changed_blockhints(hashes.get(), hashes_size, filerelpath, orifd, blockcount, changedblocks, bhmapdata) == -1) + return -1; + } + + close(orifd); if (write_blockhashmap(bhmapfile, hashes.get(), hashes_size) == -1) return -1; - if (update_hashtree_entry(parentdirhash, !bhmapdata.empty(), oldfilehash, hashes[0], bhmapfile, relpath) == -1) + if (update_hashtree_entry(parentdirhash, !bhmapdata.empty(), oldfilehash, hashes[0], bhmapfile, filerelpath) == -1) return -1; return 0; @@ -151,7 +162,7 @@ int hashmap_builder::get_blockindex(std::map &idxmap, uin return 0; } -int hashmap_builder::update_hashes( +int hashmap_builder::update_hashes_with_backup_blockhints( hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, const uint32_t blockcount, const uint32_t original_blockcount, const std::map &bindex, const std::vector &bhmapdata) { @@ -193,7 +204,58 @@ int hashmap_builder::update_hashes( } // Calculate the new file hash: filehash = HASH(filename + XOR(block hashes)) - hasher::B2H filehash{0, 0, 0, 0}; + hasher::B2H filehash = hasher::B2H_empty; + for (int i = 1; i <= blockcount; i++) + filehash ^= hashes[i]; + + // Rehash the file hash with filename included. + const std::string filename = boost::filesystem::path(relpath.data()).filename().string(); + filehash = hasher::hash(filename.c_str(), filename.length(), &filehash, hasher::HASH_SIZE); + + hashes[0] = filehash; + return 0; +} + +int hashmap_builder::update_hashes_with_changed_blockhints( + hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, + const uint32_t blockcount, const std::map &bindex, const std::vector &bhmapdata) +{ + // If both existing delta block index and block hash map is available, we can just overlay the + // changed block hashes (mentioned in the delta block index) on top of the old block hashes. + // This would prevent unncessarily hashing lot of blocks. + + if (!bindex.empty()) + { + // Load old hashes if exists. + if (!bhmapdata.empty()) + memcpy(hashes, bhmapdata.data(), hashes_size < bhmapdata.size() ? hashes_size : bhmapdata.size()); + + // Refer to the block index and overlay the new hash into the hashes array. + for (const auto [blockid, newhash] : bindex) + hashes[blockid + 1] = newhash; + + // If the block hash map didn't existed, we need to calculate and fill the unchanged block hashes from the actual file. + if (bhmapdata.empty()) + { + for (uint32_t blockid = 0; blockid < blockcount; blockid++) + { + if (bindex.count(blockid) == 0 && compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1) + return -1; + } + } + } + else + { + // If we don't have the changed block index, we have to hash the entire file blocks again. + for (uint32_t blockid = 0; blockid < blockcount; blockid++) + { + if (compute_blockhash(hashes[blockid + 1], blockid, orifd, relpath) == -1) + return -1; + } + } + + // Calculate the new file hash: filehash = HASH(filename + XOR(block hashes)) + hasher::B2H filehash = hasher::B2H_empty; for (int i = 1; i <= blockcount; i++) filehash ^= hashes[i]; diff --git a/src/statefs/hashmap_builder.hpp b/src/statefs/hashmap_builder.hpp index 2aafc441..53198d31 100644 --- a/src/statefs/hashmap_builder.hpp +++ b/src/statefs/hashmap_builder.hpp @@ -1,5 +1,5 @@ -#ifndef _STATEFS_HASHMAP_BUILDER_ -#define _STATEFS_HASHMAP_BUILDER_ +#ifndef _HP_STATEFS_HASHMAP_BUILDER_ +#define _HP_STATEFS_HASHMAP_BUILDER_ #include "../pchheader.hpp" #include "hasher.hpp" @@ -17,16 +17,19 @@ private: int read_blockhashmap(std::vector &bhmapdata, std::string &hmapfile, const std::string &relpath); int get_blockindex(std::map &idxmap, uint32_t &totalblockcount, const std::string &filerelpath); - int update_hashes( + int update_hashes_with_backup_blockhints( hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, const uint32_t blockcount, const uint32_t original_blockcount, const std::map &bindex, const std::vector &bhmapdata); + int update_hashes_with_changed_blockhints( + hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, + const uint32_t blockcount, const std::map &bindex, const std::vector &bhmapdata); int compute_blockhash(hasher::B2H &hash, uint32_t blockid, int filefd, const std::string &relpath); int write_blockhashmap(const std::string &bhmapfile, const hasher::B2H *hashes, const off_t hashes_size); int update_hashtree_entry(hasher::B2H &parentdirhash, const bool oldbhmap_exists, const hasher::B2H oldfilehash, const hasher::B2H newfilehash, const std::string &bhmapfile, const std::string &relpath); public: hashmap_builder(const statedir_context &ctx); - int generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath); + int generate_hashmap_forfile(hasher::B2H &parentdirhash, const std::string &filepath, const std::string filerelpath, const std::map &changedblocks); int remove_hashmapfile(hasher::B2H &parentdirhash, const std::string &filepath); }; diff --git a/src/statefs/hashtree_builder.cpp b/src/statefs/hashtree_builder.cpp index fd3d1fcd..93607d81 100644 --- a/src/statefs/hashtree_builder.cpp +++ b/src/statefs/hashtree_builder.cpp @@ -8,15 +8,51 @@ namespace statefs hashtree_builder::hashtree_builder(const statedir_context &ctx) : ctx(ctx), hmapbuilder(ctx) { + force_rebuild_all = false; + hintmode = false; } int hashtree_builder::generate(hasher::B2H &roothash) { // Load modified file path hints if available. - populate_hintpaths(IDX_TOUCHEDFILES); - populate_hintpaths(IDX_NEWFILES); + populate_hintpaths_from_idxfile(IDX_TOUCHEDFILES); + populate_hintpaths_from_idxfile(IDX_NEWFILES); hintmode = !hintpaths.empty(); + return traverse_and_generate(roothash); +} + +int hashtree_builder::generate(hasher::B2H &roothash, const bool force_all) +{ + force_rebuild_all = force_all; + if (force_rebuild_all) + { + boost::filesystem::remove_all(ctx.blockhashmapdir); + boost::filesystem::remove_all(ctx.hashtreedir); + + boost::filesystem::create_directories(ctx.blockhashmapdir); + boost::filesystem::create_directories(ctx.hashtreedir); + } + + return traverse_and_generate(roothash); +} + +int hashtree_builder::generate(hasher::B2H &roothash, const std::unordered_map> &touchedfiles) +{ + hintmode = true; + fileblockindex = touchedfiles; + for (const auto &[relpath, bindex] : touchedfiles) + insert_hintpath(relpath); + + return traverse_and_generate(roothash); +} + +int hashtree_builder::traverse_and_generate(hasher::B2H &roothash) +{ + // Load current root hash if exist. + const std::string dirhashfile = ctx.hashtreedir + "/" + DIRHASH_FNAME; + roothash = get_existingdirhash(dirhashfile); + traversel_rootdir = ctx.datadir; removal_mode = false; if (update_hashtree(roothash) != 0) @@ -117,6 +153,10 @@ int hashtree_builder::update_hashtree_fordir(hasher::B2H &parentdirhash, const s parentdirhash ^= original_dirhash; parentdirhash ^= dirhash; } + else + { + parentdirhash = dirhash; + } return 0; } @@ -124,7 +164,7 @@ int hashtree_builder::update_hashtree_fordir(hasher::B2H &parentdirhash, const s hasher::B2H hashtree_builder::get_existingdirhash(const std::string &dirhashfile) { // Load current dir hash if exist. - hasher::B2H dirhash{0, 0, 0, 0}; + hasher::B2H dirhash = hasher::B2H_empty; int dirhashfd = open(dirhashfile.c_str(), O_RDONLY); if (dirhashfd > 0) { @@ -152,11 +192,17 @@ int hashtree_builder::save_dirhash(const std::string &dirhashfile, hasher::B2H d inline bool hashtree_builder::should_process_dir(hintpath_map::iterator &dir_itr, const std::string &dirpath) { + if (force_rebuild_all) + return true; + return (hintmode ? get_hinteddir_match(dir_itr, dirpath) : true); } bool hashtree_builder::should_process_file(const hintpath_map::iterator hintdir_itr, const std::string filepath) { + if (force_rebuild_all) + return true; + if (hintmode) { if (hintdir_itr == hintpaths.end()) @@ -191,7 +237,10 @@ int hashtree_builder::process_file(hasher::B2H &parentdirhash, const std::string created_htreesubdirs.emplace(htreedirpath); } - if (hmapbuilder.generate_hashmap_forfile(parentdirhash, filepath) == -1) + std::string relpath = get_relpath(filepath, ctx.datadir); + std::map changedblocks = fileblockindex[relpath]; + + if (hmapbuilder.generate_hashmap_forfile(parentdirhash, filepath, relpath, changedblocks) == -1) return -1; } else @@ -203,20 +252,24 @@ int hashtree_builder::process_file(hasher::B2H &parentdirhash, const std::string return 0; } -void hashtree_builder::populate_hintpaths(const char *const idxfile) +void hashtree_builder::populate_hintpaths_from_idxfile(const char *const idxfile) { std::ifstream infile(std::string(ctx.deltadir).append(idxfile)); if (!infile.fail()) { for (std::string relpath; std::getline(infile, relpath);) - { - std::string parentdir = boost::filesystem::path(relpath).parent_path().string(); - hintpaths[parentdir].emplace(relpath); - } + insert_hintpath(relpath); infile.close(); } } +void hashtree_builder::insert_hintpath(const std::string &relpath) +{ + boost::filesystem::path p_relpath(relpath); + std::string parentdir = p_relpath.parent_path().string(); + hintpaths[parentdir].emplace(relpath); +} + bool hashtree_builder::get_hinteddir_match(hintpath_map::iterator &matchitr, const std::string &dirpath) { // First check whether there's an exact match. If not check for a partial match. diff --git a/src/statefs/hashtree_builder.hpp b/src/statefs/hashtree_builder.hpp index f9819a68..0f97f79f 100644 --- a/src/statefs/hashtree_builder.hpp +++ b/src/statefs/hashtree_builder.hpp @@ -1,5 +1,5 @@ -#ifndef _STATEFS_HASHTREE_BUILDER_ -#define _STATEFS_HASHTREE_BUILDER_ +#ifndef _HP_STATEFS_HASHTREE_BUILDER_ +#define _HP_STATEFS_HASHTREE_BUILDER_ #include "../pchheader.hpp" #include "hasher.hpp" @@ -19,13 +19,16 @@ private: // Hint path map with parent dir as key and list of file paths under each parent dir. hintpath_map hintpaths; + bool force_rebuild_all; bool hintmode; bool removal_mode; std::string traversel_rootdir; + std::unordered_map> fileblockindex; // List of new root hash map sub directories created during the session. std::unordered_set created_htreesubdirs; + int traverse_and_generate(hasher::B2H &roothash); int update_hashtree(hasher::B2H &roothash); int update_hashtree_fordir(hasher::B2H &parentdirhash, const std::string &relpath, const hintpath_map::iterator hintdir_itr, const bool isrootlevel); @@ -35,12 +38,15 @@ private: bool should_process_file(const hintpath_map::iterator hintdir_itr, const std::string filepath); int process_file(hasher::B2H &parentdirhash, const std::string &filepath, const std::string &htreedirpath); int update_hashtree_entry(hasher::B2H &parentdirhash, const bool oldbhmap_exists, const hasher::B2H oldfilehash, const hasher::B2H newfilehash, const std::string &bhmapfile, const std::string &relpath); - void populate_hintpaths(const char *const idxfile); + void populate_hintpaths_from_idxfile(const char *const idxfile); + void insert_hintpath(const std::string &relpath); bool get_hinteddir_match(hintpath_map::iterator &matchitr, const std::string &dirpath); public: hashtree_builder(const statedir_context &ctx); int generate(hasher::B2H &roothash); + int generate(hasher::B2H &roothash, const bool force_all); + int generate(hasher::B2H &roothash, const std::unordered_map> &touchedfiles); }; } // namespace statefs diff --git a/src/statefs/state_common.cpp b/src/statefs/state_common.cpp index a55bbc0a..33fe972c 100644 --- a/src/statefs/state_common.cpp +++ b/src/statefs/state_common.cpp @@ -6,13 +6,14 @@ namespace statefs { std::string statehistdir; +statedir_context current_ctx; void init(const std::string &statehist_dir_root) { statehistdir = realpath(statehist_dir_root.c_str(), NULL); // Initialize 0 state (current state) directory. - get_statedir_context(0, true); + current_ctx = get_statedir_context(0, true); } std::string get_statedir_root(const int16_t checkpointid) diff --git a/src/statefs/state_common.hpp b/src/statefs/state_common.hpp index 64dc00ea..c1411087 100644 --- a/src/statefs/state_common.hpp +++ b/src/statefs/state_common.hpp @@ -1,5 +1,5 @@ -#ifndef _STATEFS_STATE_COMMON_ -#define _STATEFS_STATE_COMMON_ +#ifndef _HP_STATEFS_STATE_COMMON_ +#define _HP_STATEFS_STATE_COMMON_ #include #include @@ -8,6 +8,7 @@ namespace statefs { +// Max number of state history checkpoints to keep. constexpr int16_t MAX_CHECKPOINTS = 5; // Cache block size. @@ -15,7 +16,6 @@ constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB // Cache block index entry bytes length. constexpr size_t BLOCKINDEX_ENTRY_SIZE = 44; -constexpr size_t MAX_HASHES = BLOCK_SIZE / hasher::HASH_SIZE; // Permissions used when creating block cache and index files. constexpr int FILE_PERMS = 0644; @@ -38,17 +38,24 @@ const char *const BHMAP_DIR = "/bhmap"; const char *const HTREE_DIR = "/htree"; const char *const DELTA_DIR = "/delta"; -extern std::string statehistdir; - +/** + * Context struct to hold all state-related directory paths. + */ struct statedir_context { - std::string rootdir; - std::string datadir; - std::string blockhashmapdir; - std::string hashtreedir; - std::string deltadir; + std::string rootdir; // Directory holding state sub dirs. + std::string datadir; // Directory containing smart contract data. + std::string blockhashmapdir; // Directory containing block hash map files. + std::string hashtreedir; // Directory containing hash tree files (dir.hash and hard links). + std::string deltadir; // Directory containing original smart contract data. }; +// Container directory to contain all checkpoints. +extern std::string statehistdir; + +// Currently loaded state checkpoint directory context (usually checkpoint 0) +extern statedir_context current_ctx; + void init(const std::string &statehist_dir_root); std::string get_statedir_root(const int16_t checkpointid); statedir_context get_statedir_context(int16_t checkpointid = 0, bool createdirs = false); diff --git a/src/statefs/state_monitor/fusefs.cpp b/src/statefs/state_monitor/fusefs.cpp index e9e768cf..1f1fcd44 100644 --- a/src/statefs/state_monitor/fusefs.cpp +++ b/src/statefs/state_monitor/fusefs.cpp @@ -553,6 +553,7 @@ static void sfs_rename(fuse_req_t req, fuse_ino_t parent, const char *name, return; } + // state monitor hook. std::string oldfilepath, newfilepath; if (helpers::getfilepath(oldfilepath, inode_p.fd, name) == 0 && helpers::getfilepath(newfilepath, inode_np.fd, newname) == 0) @@ -568,6 +569,7 @@ static void sfs_unlink(fuse_req_t req, fuse_ino_t parent, const char *name) { Inode &inode_p = get_inode(parent); + // state monitor hook. std::string filepath; if (helpers::getfilepath(filepath, inode_p.fd, name) == 0) statemonitor.ondelete(filepath); @@ -867,6 +869,7 @@ static void sfs_create(fuse_req_t req, fuse_ino_t parent, const char *name, } else { + // state monitor hook. statemonitor.oncreate(fd); fuse_reply_create(req, &e, fi); } @@ -911,6 +914,7 @@ static void sfs_open(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) char buf[64]; sprintf(buf, "/proc/self/fd/%i", inode.fd); + // state monitor hook. statemonitor.onopen(inode.fd, fi->flags); auto fd = open(buf, fi->flags & ~O_NOFOLLOW); @@ -933,7 +937,10 @@ static void sfs_release(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) { (void)ino; close(fi->fh); + + // state monitor hook. statemonitor.onclose(fi->fh); + fuse_reply_err(req, 0); } @@ -997,6 +1004,7 @@ static void sfs_write_buf(fuse_req_t req, fuse_ino_t ino, fuse_bufvec *in_buf, (void)ino; auto size{fuse_buf_size(in_buf)}; + // state monitor hook. statemonitor.onwrite(fi->fh, off, size); do_write_buf(req, size, off, in_buf, fi); @@ -1251,7 +1259,14 @@ void maximize_fd_limit() warn("WARNING: setrlimit() failed with"); } -int start(const char *arg0, const char *statehistdir, const char *fusemntdir) +/** + * Starts hosting the fuse file system along with the state monitor. + * @param arg0 First CLI argument to be passed into fuse main. + * @param state_hist_dir Hot pocket state history directory. + * @param fuse_mnt_dir Directory to mound the fuse filesystem. + * @return 0 on success. 1 on failure. + */ +int start(const char *arg0, const char *state_hist_dir, const char *fuse_mnt_dir) { // We need an fd for every entry in our the filesystem that the // kernel knows about. This is way more than most processes need, @@ -1259,14 +1274,14 @@ int start(const char *arg0, const char *statehistdir, const char *fusemntdir) maximize_fd_limit(); // We consider this as the first run of the history dir is empty. - const bool firstrun = boost::filesystem::is_empty(statehistdir); + const bool is_first_run = boost::filesystem::is_empty(state_hist_dir); - statefs::init(statehistdir); + statefs::init(state_hist_dir); statemonitor.ctx = statefs::get_statedir_context(); fs.source = statemonitor.ctx.datadir; // Create a checkpoint from the second run onwards. - if (!firstrun) + if (!is_first_run) statemonitor.create_checkpoint(); // Initialize filesystem root @@ -1311,7 +1326,7 @@ int start(const char *arg0, const char *statehistdir, const char *fusemntdir) struct fuse_loop_config loop_config; loop_config.clone_fd = 0; loop_config.max_idle_threads = 10; - if (fuse_session_mount(se, fusemntdir) != 0) + if (fuse_session_mount(se, fuse_mnt_dir) != 0) goto err_out3; ret = fuse_session_loop_mt(se, &loop_config); @@ -1338,5 +1353,5 @@ int main(int argc, char *argv[]) exit(1); } - fusefs::start(argv[0], argv[1], argv[2]); + return fusefs::start(argv[0], argv[1], argv[2]); } \ No newline at end of file diff --git a/src/statefs/state_monitor/state_monitor.cpp b/src/statefs/state_monitor/state_monitor.cpp index 46d57736..d943beaa 100644 --- a/src/statefs/state_monitor/state_monitor.cpp +++ b/src/statefs/state_monitor/state_monitor.cpp @@ -19,11 +19,22 @@ namespace statefs { +/** + * Creates a new checkpoint directory. This will remove the oldest checkpoint if we have + * reached MAX_CHECKPOINTS. This is called whenever fuse filesystem is run so the contract + * always runs on a new checkpoint. + */ void state_monitor::create_checkpoint() { - // Shift -1 and below checkpoints by 1 more. + /** + * Checkpoints are numbered 0, -1, -2, ... + * Checkpoint 0 is the latest state containing "state", "data", "delta", "bhmap", "htree" directories. + * Checkpoints -1 and lower cotnains only the "delta" dirs containing older state changesets. + */ + + // Shift "-1" and older checkpoints by 1 more. And then copy checkpoint 0 delta dir to "-1" // If MAX oldest checkpoint is there, remove it and work our way upwards. - int16_t oldest_chkpnt = (MAX_CHECKPOINTS + 1) * -1; // +1 because we maintain one extra checkpoint in case of rollbacks. + int16_t oldest_chkpnt = MAX_CHECKPOINTS * -1; for (int16_t chkpnt = oldest_chkpnt; chkpnt <= -1; chkpnt++) { std::string dir = get_statedir_root(chkpnt); @@ -36,8 +47,8 @@ void state_monitor::create_checkpoint() } else { - std::string dirshift = get_statedir_root(chkpnt - 1); - boost::filesystem::rename(dir, dirshift); + std::string dir_shift = get_statedir_root(chkpnt - 1); + boost::filesystem::rename(dir, dir_shift); } } @@ -57,6 +68,10 @@ void state_monitor::create_checkpoint() return; } +/** + * Called whenever a new file is created in the fuse fs. + * @param fd The fd of the created file. + */ void state_monitor::oncreate(const int fd) { std::lock_guard lock(monitor_mutex); @@ -66,27 +81,46 @@ void state_monitor::oncreate(const int fd) oncreate_filepath(filepath); } +/** + * Called whenever a file is going to be opened. + * @param inodefd inode fd given by fuse fs. This is used to find the physical path of the file. + * @param flags Open flags. + */ void state_monitor::onopen(const int inodefd, const int flags) { std::lock_guard lock(monitor_mutex); + // Find the actual file path which is going to be opened and add that path to tracked file info list. std::string filepath; if (extract_filepath(filepath, inodefd) == 0) { state_file_info *fi; if (get_tracked_fileinfo(&fi, filepath) == 0) { - // Check whether fd is open in truncate mode. If so cache the entire file immediately. + // Check whether the file is going to be opened in truncate mode. + // If so cache the entire file immediately because this is the last chance we get to backup the data. if (flags & O_TRUNC) cache_blocks(*fi, 0, fi->original_length); } } } +/** + * Called whenever a file is being written to. + * @param fd fd of the file being written to. + * @param offset Byte offset of the write. + * @param length Number of bytes being overwritten. + */ void state_monitor::onwrite(const int fd, const off_t offset, const size_t length) { + // TODO: Known issue: Onwrite can get called if the client program deletes a file before + // closing the currently open file. If there were some bytes on the write buffer, the flush happens + // when the client closes the fd. By that time the fd is invalid since the file is deleted. + // However nothing happens to us as our code simply returns on invalild fd error. + std::lock_guard lock(monitor_mutex); + // Find the actual filepath being written to and cache the blocks to server as backup. std::string filepath; if (get_fd_filepath(filepath, fd) == 0) { @@ -96,20 +130,30 @@ void state_monitor::onwrite(const int fd, const off_t offset, const size_t lengt } } -void state_monitor::onrename(const std::string &oldfilepath, const std::string &newfilepath) +/** + * Called when a file is being renamed. + * We simply treat this as delete-and-create operation. + */ +void state_monitor::onrename(const std::string &old_filepath, const std::string &new_filepath) { std::lock_guard lock(monitor_mutex); - ondelete_filepath(oldfilepath); - oncreate_filepath(newfilepath); + ondelete_filepath(old_filepath); + oncreate_filepath(new_filepath); } +/** + * Called when a file is being deleted. + */ void state_monitor::ondelete(const std::string &filepath) { std::lock_guard lock(monitor_mutex); ondelete_filepath(filepath); } +/** + * Called when a file is being truncated. + */ void state_monitor::ontruncate(const int fd, const off_t newsize) { std::lock_guard lock(monitor_mutex); @@ -124,19 +168,25 @@ void state_monitor::ontruncate(const int fd, const off_t newsize) } } +/** + * Called when an open file is being closed. Here, we clear any tracking information we kept for this file + * and close off any related fds associated with any backup operations for this file. + */ void state_monitor::onclose(const int fd) { std::lock_guard lock(monitor_mutex); - auto pitr = fdpathmap.find(fd); - if (pitr != fdpathmap.end()) + // fd_path_map should contain this fd already if we were tracking it. + + auto pitr = fd_path_map.find(fd); + if (pitr != fd_path_map.end()) { // Close any block cache/index fds we have opened for this file. - auto fitr = fileinfomap.find(pitr->second); // pitr->second is the filepath string. - if (fitr != fileinfomap.end()) - close_cachingfds(fitr->second); // fitr->second is the fileinfo struct. + auto fitr = file_info_map.find(pitr->second); // pitr->second is the filepath string. + if (fitr != file_info_map.end()) + close_caching_fds(fitr->second); // fitr->second is the fileinfo struct. - fdpathmap.erase(pitr); + fd_path_map.erase(pitr); } } @@ -170,8 +220,8 @@ int state_monitor::extract_filepath(std::string &filepath, const int fd) int state_monitor::get_fd_filepath(std::string &filepath, const int fd) { // Return path from the map if found. - const auto itr = fdpathmap.find(fd); - if (itr != fdpathmap.end()) + const auto itr = fd_path_map.find(fd); + if (itr != fd_path_map.end()) { filepath = itr->second; return 0; @@ -180,42 +230,51 @@ int state_monitor::get_fd_filepath(std::string &filepath, const int fd) // Extract the file path and populate the fd-->filepath map. if (extract_filepath(filepath, fd) == 0) { - fdpathmap[fd] = filepath; + fd_path_map[fd] = filepath; return 0; } return -1; } +/** + * Called when a new file is going to be created. fd is not yet open at this point. + * We need to catch this and start tracking this filepath. + */ void state_monitor::oncreate_filepath(const std::string &filepath) { // Check whether we are already tracking this file path. - // Only way this could happen is deleting an existing file and creating a new file with same name. - if (fileinfomap.count(filepath) == 0) + // Only way we could be tracking this patth already is deleting an existing file and creating + // a new file with same name. + if (file_info_map.count(filepath) == 0) { // Add an entry for the new file in the file info map. This information will be used to ignore // future operations (eg. write/delete) done to this file. state_file_info fi; - fi.isnew = true; + fi.is_new = true; fi.filepath = filepath; - fileinfomap[filepath] = std::move(fi); + file_info_map[filepath] = std::move(fi); // Add to the list of new files added during this session. - write_newfileentry(filepath); + write_new_file_entry(filepath); } } +/** + * Called when a file is going to be deleted. We use this to remove any tracking information + * regarding this file and to backup the file before deletion. + */ void state_monitor::ondelete_filepath(const std::string &filepath) { state_file_info *fi; if (get_tracked_fileinfo(&fi, filepath) == 0) { - if (fi->isnew) + if (fi->is_new) { // If this is a new file, just remove from existing index entries. // No need to cache the file blocks. - remove_newfileentry(fi->filepath); - fileinfomap.erase(filepath); + remove_new_file_entry(fi->filepath); + file_info_map.erase(filepath); } else { @@ -234,15 +293,15 @@ void state_monitor::ondelete_filepath(const std::string &filepath) int state_monitor::get_tracked_fileinfo(state_file_info **fi, const std::string &filepath) { // Return from filepath-->fileinfo map if found. - const auto itr = fileinfomap.find(filepath); - if (itr != fileinfomap.end()) + const auto itr = file_info_map.find(filepath); + if (itr != file_info_map.end()) { *fi = &itr->second; return 0; } // Initialize a new state file info struct for the given filepath. - state_file_info &fileinfo = fileinfomap[filepath]; + state_file_info &fileinfo = file_info_map[filepath]; // We use stat() to find out the length of the file. struct stat stat_buf; @@ -259,7 +318,8 @@ int state_monitor::get_tracked_fileinfo(state_file_info **fi, const std::string } /** - * Caches the specified bytes range of the given file. + * Backs up the specified bytes range of the given file. This is called whenever a file is being + * overwritten/deleted. * @param fi The file info struct pointing to the file to be cached. * @param offset The start byte position for caching. * @param length How many bytes to cache. @@ -268,7 +328,7 @@ int state_monitor::get_tracked_fileinfo(state_file_info **fi, const std::string int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const size_t length) { // No caching required if this is a new file created during this session. - if (fi.isnew) + if (fi.is_new) return 0; uint32_t original_blockcount = ceil((double)fi.original_length / (double)BLOCK_SIZE); @@ -277,7 +337,7 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s if (original_blockcount == fi.cached_blockids.size()) return 0; - // Initialize fds and indexes required for caching. + // Initialize fds and indexes required for caching the file. if (prepare_caching(fi) != 0) return -1; @@ -291,7 +351,8 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s // std::cout << "Cache blocks: '" << fi.filepath << "' [" << offset << "," << length << "] " << startblock << "," << endblock << "\n"; // If this is the first time we are caching this file, write an entry to the touched file index. - if (fi.cached_blockids.empty() && write_touchedfileentry(fi.filepath) != 0) + // Touched file index is used by rollback to server as a guide. + if (fi.cached_blockids.empty() && write_touched_file_entry(fi.filepath) != 0) return -1; for (uint32_t i = startblock; i <= endblock; i++) @@ -326,6 +387,7 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s // Whoever is using the index must sort it if required. // Entry format: [blocknum(4 bytes) | cacheoffset(8 bytes) | blockhash(32 bytes)] + // Calculate the block hash by combining block offset with block data. char entrybuf[BLOCKINDEX_ENTRY_SIZE]; hasher::B2H hash = hasher::hash(&blockoffset, 8, blockbuf.get(), bytesread); @@ -350,13 +412,13 @@ int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const s } /** - * Initializes fds and indexes required for caching. + * Initializes fds and indexes required for caching a particular file. * @param fi The state file info struct pointing to the file being cached. * @return 0 on succesful initialization. -1 on failure. */ int state_monitor::prepare_caching(state_file_info &fi) { - // If readfd is greater than 0 then we take it as caching being already initialized. + // If readfd is greater than 0 then we take it as caching being already initialized for this file. if (fi.readfd > 0) return 0; @@ -378,10 +440,10 @@ int state_monitor::prepare_caching(state_file_info &fi) // Create directory tree if not exist so we are able to create the cache and index files. boost::filesystem::path cachesubdir = boost::filesystem::path(tmppath).parent_path(); - if (created_cachesubdirs.count(cachesubdir.string()) == 0) + if (created_cache_subdirs.count(cachesubdir.string()) == 0) { boost::filesystem::create_directories(cachesubdir); - created_cachesubdirs.emplace(cachesubdir.string()); + created_cache_subdirs.emplace(cachesubdir.string()); } // Create and open the block cache file. @@ -415,7 +477,7 @@ int state_monitor::prepare_caching(state_file_info &fi) /** * Closes any open caching fds for a given file. */ -void state_monitor::close_cachingfds(state_file_info &fi) +void state_monitor::close_caching_fds(state_file_info &fi) { if (fi.readfd > 0) close(fi.readfd); @@ -433,25 +495,25 @@ void state_monitor::close_cachingfds(state_file_info &fi) /** * Inserts a file into the modified files list of this session. - * This index is used to restore modified files during restore. + * This index is used to restore modified files during rollback. */ -int state_monitor::write_touchedfileentry(std::string_view filepath) +int state_monitor::write_touched_file_entry(std::string_view filepath) { - if (touchedfileindexfd <= 0) + if (touched_fileindex_fd <= 0) { - std::string indexfile = ctx.deltadir + "/idxtouched.idx"; - touchedfileindexfd = open(indexfile.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); - if (touchedfileindexfd <= 0) + std::string index_file = ctx.deltadir + IDX_TOUCHEDFILES; + touched_fileindex_fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); + if (touched_fileindex_fd <= 0) { - std::cerr << errno << ": Open failed " << indexfile << "\n"; + std::cerr << errno << ": Open failed " << index_file << "\n"; return -1; } } // Write the relative file path line to the index. filepath = filepath.substr(ctx.datadir.length(), filepath.length() - ctx.datadir.length()); - write(touchedfileindexfd, filepath.data(), filepath.length()); - write(touchedfileindexfd, "\n", 1); + write(touched_fileindex_fd, filepath.data(), filepath.length()); + write(touched_fileindex_fd, "\n", 1); return 0; } @@ -459,13 +521,13 @@ int state_monitor::write_touchedfileentry(std::string_view filepath) * Inserts a file into the list of new files created during this session. * This index is used in deleting new files during restore. */ -int state_monitor::write_newfileentry(std::string_view filepath) +int state_monitor::write_new_file_entry(std::string_view filepath) { - std::string indexfile = ctx.deltadir + "/idxnew.idx"; - int fd = open(indexfile.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); + std::string index_file = ctx.deltadir + IDX_NEWFILES; + int fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); if (fd <= 0) { - std::cerr << errno << ": Open failed " << indexfile << "\n"; + std::cerr << errno << ": Open failed " << index_file << "\n"; return -1; } @@ -479,27 +541,28 @@ int state_monitor::write_newfileentry(std::string_view filepath) /** * Scans and removes the given filepath from the new files index. + * This is called when a file added during this session gets deleted in the same session. */ -void state_monitor::remove_newfileentry(std::string_view filepath) +void state_monitor::remove_new_file_entry(std::string_view filepath) { filepath = filepath.substr(ctx.datadir.length(), filepath.length() - ctx.datadir.length()); // We create a copy of the new file index and transfer lines from first file // to the second file except the line matching the given filepath. - std::string indexfile = ctx.deltadir + "/idxnew.idx"; - std::string indexfile_tmp = ctx.deltadir + "/idxnew.idx.tmp"; + std::string index_file = ctx.deltadir + IDX_NEWFILES; + std::string index_file_tmp = ctx.deltadir + IDX_NEWFILES + ".tmp"; - std::ifstream infile(indexfile); - std::ofstream outfile(indexfile_tmp); + std::ifstream infile(index_file); + std::ofstream outfile(index_file_tmp); - bool linestransferred = false; + bool lines_transferred = false; for (std::string line; std::getline(infile, line);) { if (line != filepath) // Skip the file being removed. { outfile << line << "\n"; - linestransferred = true; + lines_transferred = true; } } @@ -507,13 +570,13 @@ void state_monitor::remove_newfileentry(std::string_view filepath) outfile.close(); // Remove the old index. - std::remove(indexfile.c_str()); + std::remove(index_file.c_str()); // If no lines transferred, delete the temp file as well. - if (linestransferred) - std::rename(indexfile_tmp.c_str(), indexfile.c_str()); + if (lines_transferred) + std::rename(index_file_tmp.c_str(), index_file.c_str()); else - std::remove(indexfile_tmp.c_str()); + std::remove(index_file_tmp.c_str()); } } // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_monitor/state_monitor.hpp b/src/statefs/state_monitor/state_monitor.hpp index 7958bdc8..6d802136 100644 --- a/src/statefs/state_monitor/state_monitor.hpp +++ b/src/statefs/state_monitor/state_monitor.hpp @@ -1,5 +1,5 @@ -#ifndef _STATEFS_STATE_MONITOR_ -#define _STATEFS_STATE_MONITOR_ +#ifndef _HP_STATEFS_STATE_MONITOR_ +#define _HP_STATEFS_STATE_MONITOR_ #include #include @@ -12,40 +12,41 @@ namespace statefs { -// Holds information about an original file in state that we are tracking. +/** + * Holds information about an original file in state that we are tracking. + */ struct state_file_info { - bool isnew; - off_t original_length; - std::unordered_set cached_blockids; - std::string filepath; - int readfd; - int cachefd; - int indexfd; + bool is_new; // Whether this is a new file created during this session. + off_t original_length; // Original file length. + std::unordered_set cached_blockids; // Set of block ids cached during this session. + std::string filepath; // Actual real path of the file. (not fuse path) + int readfd; // fd used for reading the original file for caching. + int cachefd; // fd for writing into the block cache file. + int indexfd; // fd for writing into the block index file. }; -// Invoked by fuse file system for relevent file system calls. +/** + * Invoked by fuse file system for relevent file system calls. + */ class state_monitor { private: // Map of fd-->filepath - std::unordered_map fdpathmap; + std::unordered_map fd_path_map; // Map of filepath-->fileinfo - std::unordered_map fileinfomap; - - // Complete list of modified files during the session. - std::unordered_set touchedfiles; + std::unordered_map file_info_map; // List of new cache sub directories created during the session. - std::unordered_set created_cachesubdirs; + std::unordered_set created_cache_subdirs; // Mutex to synchronize parallel file system calls into our custom state tracking logic. std::mutex monitor_mutex; // Holds the fd used to write into modified files index. This will be kept open for the entire // life of the state monitor. - int touchedfileindexfd = 0; + int touched_fileindex_fd = 0; int extract_filepath(std::string &filepath, const int fd); int get_fd_filepath(std::string &filepath, const int fd); @@ -55,10 +56,10 @@ private: int cache_blocks(state_file_info &fi, const off_t offset, const size_t length); int prepare_caching(state_file_info &fi); - void close_cachingfds(state_file_info &fi); - int write_touchedfileentry(std::string_view filepath); - int write_newfileentry(std::string_view filepath); - void remove_newfileentry(std::string_view filepath); + void close_caching_fds(state_file_info &fi); + int write_touched_file_entry(std::string_view filepath); + int write_new_file_entry(std::string_view filepath); + void remove_new_file_entry(std::string_view filepath); public: statedir_context ctx; @@ -66,7 +67,7 @@ public: void oncreate(const int fd); void onopen(const int inodefd, const int flags); void onwrite(const int fd, const off_t offset, const size_t length); - void onrename(const std::string &oldfilepath, const std::string &newfilepath); + void onrename(const std::string &old_filepath, const std::string &new_filepath); void ondelete(const std::string &filepath); void ontruncate(const int fd, const off_t newsize); void onclose(const int fd); diff --git a/src/statefs/state_restore.cpp b/src/statefs/state_restore.cpp index f856a085..d597e841 100644 --- a/src/statefs/state_restore.cpp +++ b/src/statefs/state_restore.cpp @@ -20,7 +20,7 @@ void state_restore::delete_newfiles() std::string filepath(ctx.datadir); filepath.append(file); - std::remove(filepath.c_str()); + remove(filepath.c_str()); } infile.close(); diff --git a/src/statefs/state_restore.hpp b/src/statefs/state_restore.hpp index 1038c441..c1d076d8 100644 --- a/src/statefs/state_restore.hpp +++ b/src/statefs/state_restore.hpp @@ -1,5 +1,5 @@ -#ifndef _STATEFS_STATE_RESTORE_ -#define _STATEFS_STATE_RESTORE_ +#ifndef _HP_STATEFS_STATE_RESTORE_ +#define _HP_STATEFS_STATE_RESTORE_ #include "../pchheader.hpp" #include "hasher.hpp" diff --git a/src/statefs/state_store.cpp b/src/statefs/state_store.cpp new file mode 100644 index 00000000..9bc60deb --- /dev/null +++ b/src/statefs/state_store.cpp @@ -0,0 +1,357 @@ +#include "../pchheader.hpp" +#include "hasher.hpp" +#include "state_common.hpp" +#include "hashtree_builder.hpp" +#include "state_store.hpp" +#include "../hplog.hpp" +#include "state_store.hpp" + +namespace statefs +{ + +// Map of modified/deleted files with updated blockids and hashes (if modified). +std::unordered_map> touched_files; + +/** + * Checks whether the given directory exists in the state data directory. + */ +bool is_dir_exists(const std::string &dir_relpath) +{ + const std::string full_path = current_ctx.datadir + dir_relpath; + return boost::filesystem::exists(full_path); +} + +/** + * Retrieves the hash list of the file system entries at a given directory. + * @return 0 on success. -1 on failure. + */ +int get_fs_entry_hashes(std::unordered_map &fs_entries, const std::string &dir_relpath, const hasher::B2H expected_hash) +{ + // TODO: instead of iterating the data dir, we could simply query the hash tree directory + // listing and get the hashes using the hardlink names straight away. But then we don't have + // a way to get the file names. If we could implement a mechanism for that we could make this efficient. + + if (expected_hash != hasher::B2H_empty) + { + // Check whether the existing block hash matches expected hash. + const std::string dir_hash_path = current_ctx.hashtreedir + dir_relpath + DIRHASH_FNAME; + + hasher::B2H existsing_hash; + if (read_file_bytes(&existsing_hash, dir_hash_path.c_str(), 0, hasher::HASH_SIZE) == -1) + return -1; + + if (existsing_hash != expected_hash) + return -1; + } + + const std::string full_path = current_ctx.datadir + dir_relpath; + for (const boost::filesystem::directory_entry &dentry : boost::filesystem::directory_iterator(full_path)) + { + const boost::filesystem::path p = dentry.path(); + + p2p::state_fs_hash_entry fs_entry; + fs_entry.is_file = !boost::filesystem::is_directory(p); + + std::string fsentry_relpath = dir_relpath + p.filename().string(); + + // Read the first 32 bytes of the .bhmap file or dir.hash file. + + std::string hash_path; + + if (fs_entry.is_file) + { + hash_path = current_ctx.blockhashmapdir + fsentry_relpath + HASHMAP_EXT; + } + else + { + fsentry_relpath += "/"; + hash_path = current_ctx.hashtreedir + fsentry_relpath + DIRHASH_FNAME; + // Skip the directory if it doesn't contain the dir.hash file. + // By that we assume the directory is empty so we're not interested in it. + if (!boost::filesystem::exists(hash_path)) + continue; + } + + if (read_file_bytes(&fs_entry.hash, hash_path.c_str(), 0, hasher::HASH_SIZE) == -1) + return -1; + + fs_entries.emplace(fsentry_relpath, std::move(fs_entry)); + } + return 0; +} + +/** + * Retrieves the block hash map for a file. + * @return 0 on success. -1 on failure. + */ +int get_block_hash_map(std::vector &vec, const std::string &file_relpath, const hasher::B2H expected_hash) +{ + const std::string bhmap_path = current_ctx.blockhashmapdir + file_relpath + HASHMAP_EXT; + + if (expected_hash != hasher::B2H_empty) + { + // Check whether the existing block hash matches expected hash. + + if (!boost::filesystem::exists(bhmap_path) || read_file_bytes_to_end(vec, bhmap_path.c_str(), 0) == -1) + return -1; + + // Existing hash is the first 32 bytes of bhmap contents. + hasher::B2H existing_hash = *reinterpret_cast(vec.data()); + if (existing_hash != expected_hash) + return -1; + + // Return the bhmap bytes without the first 32 bytes. + vec.erase(vec.begin(), vec.begin() + hasher::HASH_SIZE); + } + else + { + // Skip the file root hash and get the rest of the bytes. + if (boost::filesystem::exists(bhmap_path) && read_file_bytes_to_end(vec, bhmap_path.c_str(), hasher::HASH_SIZE) == -1) + return -1; + } + + return 0; +} + +/** + * Retrieves the byte length of a file. + * @return 0 on success. -1 on failure. + */ +int get_file_length(const std::string &file_relpath) +{ + std::string full_path = current_ctx.datadir + file_relpath; + int fd = open(full_path.c_str(), O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << "Open failed " << full_path; + return -1; + } + + const off_t total_len = lseek(fd, 0, SEEK_END); + close(fd); + + return total_len; +} + +/** + * Retrieves the specified data block from a state file. + * @return Number of bytes read on success. -1 on failure. + */ +int get_block(std::vector &vec, const std::string &file_relpath, const uint32_t block_id, const hasher::B2H expected_hash) +{ + // Check whether the existing block hash matches expected hash. + if (expected_hash != hasher::B2H_empty) + { + std::string bhmap_path = current_ctx.blockhashmapdir + file_relpath + HASHMAP_EXT; + hasher::B2H existing_hash = hasher::B2H_empty; + + if (read_file_bytes(&existing_hash, bhmap_path.c_str(), (block_id + 1) * hasher::HASH_SIZE, hasher::HASH_SIZE) == -1) + return -1; + + if (existing_hash != expected_hash) + return -1; + } + + std::string full_path = current_ctx.datadir + file_relpath; + vec.resize(BLOCK_SIZE); + int read_bytes = read_file_bytes(vec.data(), full_path.c_str(), block_id * BLOCK_SIZE, BLOCK_SIZE); + + if (read_bytes == -1) + return -1; + + vec.resize(read_bytes); + return read_bytes; +} + +/** + * Creates the specified directory in the state data directory. + */ +void create_dir(const std::string &dir_relpath) +{ + const std::string full_path = current_ctx.datadir + dir_relpath; + boost::filesystem::create_directories(full_path); +} + +/** + * Deletes all files within the specified state sub directory and marks the changes. + * @return 0 on success. -1 on failure. + */ +int delete_dir(const std::string &dir_relpath) +{ + std::string full_dir_path = current_ctx.datadir + dir_relpath; + + const boost::filesystem::directory_iterator itr_end; + for (boost::filesystem::directory_iterator itr(full_dir_path); itr != itr_end; itr++) + { + boost::filesystem::path p = itr->path(); + + if (!boost::filesystem::is_directory(p)) + { + if (!boost::filesystem::remove(p)) + return -1; + + // Add the deleted file rel path to the touched files list. + touched_files.emplace( + get_relpath(p.string(), current_ctx.datadir), + std::map()); + } + } + + // Finally, delete the directory itself. + boost::filesystem::remove_all(full_dir_path); + + return 0; +} + +/** + * Deletes the specified state file and marks the change. + * @return 0 on success. -1 on failure. + */ +int delete_file(const std::string &file_relpath) +{ + std::string full_path = current_ctx.datadir + file_relpath; + if (!boost::filesystem::remove(full_path)) + return -1; + + touched_files.emplace(file_relpath, std::map()); + return 0; +} + +/** + * Truncates the specified state file to the specified length and marks the change. + * @return 0 on success. -1 on failure. + */ +int truncate_file(const std::string &file_relpath, const size_t newsize) +{ + std::string full_path = current_ctx.datadir + file_relpath; + int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); + if (fd == -1) + { + LOG_ERR << errno << "Open failed " << full_path; + return -1; + } + + int ret = ftruncate(fd, newsize); + close(fd); + if (ret == -1) + { + LOG_ERR << errno << "Truncate failed " << full_path; + return -1; + } + + return 0; +} + +/** + * Writes the specified block to a file and marks the change. + * @param file_relpath State data relative path of the file. + * @param block_id Block id to replace/write. + * @param buf The buffer containing data to be written. + * @param len Length of the buffer. + * @return 0 on success. -1 on failure. + */ +int write_block(const std::string &file_relpath, const uint32_t block_id, const void *buf, const size_t len) +{ + std::string full_path = current_ctx.datadir + file_relpath; + int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); + if (fd == -1) + { + LOG_ERR << errno << "Open failed " << full_path; + return -1; + } + + const off_t offset = block_id * BLOCK_SIZE; + int ret = pwrite(fd, buf, len, offset); + close(fd); + if (ret == -1) + { + LOG_ERR << errno << "Write failed " << full_path; + return -1; + } + + hasher::B2H hash = hasher::hash(&offset, 8, buf, len); + touched_files[file_relpath].emplace(block_id, hash); + + return 0; +} + +/** + * Computes the latest hash tree with any changes recorded in touched files index. + * @return 0 on success. -1 on failure. + */ +int compute_hash_tree(hasher::B2H &statehash, const bool force_all) +{ + hashtree_builder htree_builder(current_ctx); + + int ret = force_all ? htree_builder.generate(statehash, true) : htree_builder.generate(statehash, touched_files); + + touched_files.clear(); + return ret; +} + +//-----Private helper functions---------// + +/** + * Reads bytes from file into a buffer. + * @param buf Buffer to fill with the read bytes. + * @param filepath Full path to the file. + * @param start Starting offset to read. + * @param len Number of bytes to read. + * @return Number of bytes read on successful read. -1 on failure. + */ +int read_file_bytes(void *buf, const char *filepath, const off_t start, const size_t len) +{ + int fd = open(filepath, O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << "Open failed " << filepath; + return -1; + } + + int read_bytes = pread(fd, buf, len, start); + close(fd); + if (read_bytes <= 0) + { + LOG_ERR << errno << "Read failed " << filepath; + return -1; + } + + return read_bytes; +} + +/** + * Reads bytes from file into a vector. The vector size will be adjusted to the actual bytes read. + * @param vec Vector to fill with the read bytes. + * @param filepath Full path to the file. + * @param start Starting offset to read. + * @return Number of bytes read on successful read. -1 on failure. + */ +int read_file_bytes_to_end(std::vector &vec, const char *filepath, const off_t start) +{ + int fd = open(filepath, O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << "Open failed " << filepath; + return -1; + } + + const off_t total_len = lseek(fd, 0, SEEK_END); + if (total_len == -1) + return -1; + + const size_t len = total_len - start; + vec.resize(len); + + int read_bytes = pread(fd, vec.data(), len, start); + close(fd); + if (read_bytes <= 0) + { + LOG_ERR << errno << "Read failed " << filepath; + return -1; + } + vec.resize(read_bytes); + + return read_bytes; +} + +} // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_store.hpp b/src/statefs/state_store.hpp new file mode 100644 index 00000000..fd1c0edc --- /dev/null +++ b/src/statefs/state_store.hpp @@ -0,0 +1,35 @@ +#ifndef _HP_STATEFS_STATE_STORE_ +#define _HP_STATEFS_STATE_STORE_ + +#include "../pchheader.hpp" +#include "../p2p/p2p.hpp" +#include "hasher.hpp" + +namespace statefs +{ + +// Map of modified/deleted files with updated blockids and hashes (if modified). +extern std::unordered_map> touched_files; + +bool is_dir_exists(const std::string &dir_relpath); +int get_fs_entry_hashes(std::unordered_map &fs_entries, const std::string &dir_relpath, const hasher::B2H expected_hash); +int get_block_hash_map(std::vector &vec, const std::string &file_relpath, const hasher::B2H expected_hash); +int get_file_length(const std::string &file_relpath); +int get_block(std::vector &vec, const std::string &file_relpath, const uint32_t block_id, const hasher::B2H expected_hash); +void create_dir(const std::string &dir_relpath); +int delete_dir(const std::string &dir_relpath); +int delete_file(const std::string &file_relpath); +int truncate_file(const std::string &file_relpath, const size_t newsize); +int write_block(const std::string &file_relpath, const uint32_t block_id, const void *buf, const size_t len); +int compute_hash_tree(hasher::B2H &statehash, const bool force_all = false); + +/** + * Private helper functions. + */ + +int read_file_bytes(void *buf, const char *filepath, const off_t start, const size_t len); +int read_file_bytes_to_end(std::vector &vec, const char *filepath, const off_t start); + +} // namespace statefs + +#endif \ No newline at end of file diff --git a/src/util.hpp b/src/util.hpp index f485fcf3..37b342e9 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -27,6 +27,8 @@ constexpr uint8_t MIN_PEERMSG_VERSION = 1; // (Keeping this as int for effcient msg payload and comparison) constexpr uint8_t MIN_NPL_INPUT_VERSION = 1; + + /** * FIFO hash set with a max size. */ diff --git a/test/vm-cluster/setup-hp.sh b/test/vm-cluster/setup-hp.sh index 47445f89..d719627c 100755 --- a/test/vm-cluster/setup-hp.sh +++ b/test/vm-cluster/setup-hp.sh @@ -10,14 +10,15 @@ else sudo apt-get install -y nodejs fi -if [ -x "$(command -v fusermount3)" ]; then - echo "FUSE already installed." -else +# if [ -x "$(command -v fusermount3)" ]; then +# echo "FUSE already installed." +# else echo "Installing FUSE..." sudo cp ./libfuse3.so.3 /usr/local/lib/ sudo ldconfig sudo cp ./fusermount3 /usr/local/bin/ -fi +# fi + sudo rm -r ~/contract > /dev/null 2>&1 ./hpcore new ./contract @@ -26,3 +27,18 @@ openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout tlskey.pem -ou -subj "/C=AU/ST=ST/L=L/O=O/OU=OU/CN=localhost/emailAddress=hp@example" > /dev/null 2>&1 popd > /dev/null 2>&1 +sudo mkdir -p ./contract/statehist/0 +sudo mkdir -p ./contract/statehist/0/data + +FILE=fuse-3.8.0.tar.xz +FILE2=fuse-3.8.0 +if [ -f "$FILE" ]; then + if [ -f "$FILE2" ]; then + sudo cp -r ./fuse-3.8.0 ~/contract/statehist/0/data + else + sudo tar -xf fuse-3.8.0.tar.xz -C ~/contract/statehist/0/data + fi +else +sudo wget https://github.com/libfuse/libfuse/releases/download/fuse-3.8.0/fuse-3.8.0.tar.xz - +sudo tar -xf fuse-3.8.0.tar.xz -C ~/contract/statehist/0/data +fi diff --git a/test/vm-cluster/setup-vm.sh b/test/vm-cluster/setup-vm.sh index cb4ccf02..a65b5e24 100755 --- a/test/vm-cluster/setup-vm.sh +++ b/test/vm-cluster/setup-vm.sh @@ -13,7 +13,7 @@ if [ $mode = "new" ]; then sshpass -p $vmpass scp $hpcore/build/hpcore \ $hpcore/build/hpstatemon \ $hpcore/examples/echocontract/contract.js \ - /usr/local/lib/libfuse3.so.3 \ + /usr/local/lib/x86_64-linux-gnu/libfuse3.so.3 \ /usr/local/bin/fusermount3 \ ./setup-hp.sh \ geveo@$vmip:~/