diff --git a/go.mod b/go.mod index 0ee0258..c7d0347 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/migalabs/armiarma go 1.21 require ( + github.com/attestantio/go-eth2-client v0.21.1 github.com/ethereum/go-ethereum v1.13.14 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/jackc/pgx/v4 v4.18.3 @@ -43,11 +44,14 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/gosigar v0.14.3 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/ferranbt/fastssz v0.1.3 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/goccy/go-yaml v1.9.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -98,11 +102,13 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/miekg/dns v1.1.58 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -125,6 +131,7 @@ require ( github.com/prometheus/common v0.51.1 // indirect github.com/prometheus/procfs v0.13.0 // indirect github.com/protolambda/bls12-381-util v0.1.0 // indirect + github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/quic-go/webtransport-go v0.6.0 // indirect @@ -151,8 +158,10 @@ require ( golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.19.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 0978caf..1b9fdb5 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/attestantio/go-eth2-client v0.21.1 h1:yvsMd/azPUbxiJzWZhgqfOJJRNF1zLvAJpcBXTHzyh8= +github.com/attestantio/go-eth2-client v0.21.1/go.mod h1:Tb412NpzhsC0sbtpXS4D51y5se6nDkWAi6amsJrqX9c= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= @@ -147,6 +149,11 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/ethereum/go-ethereum v1.13.14 h1:EwiY3FZP94derMCIam1iW4HFVrSgIcpsu0HwTQtm6CQ= github.com/ethereum/go-ethereum v1.13.14/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= +github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= @@ -177,11 +184,17 @@ github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/goccy/go-yaml v1.9.2 h1:2Njwzw+0+pjU2gb805ZC1B/uBuAs2VcZ3K+ZgHwDs7w= +github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -482,6 +495,7 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -722,12 +736,16 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= @@ -765,6 +783,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -957,6 +977,8 @@ github.com/protolambda/zrnt v0.32.3 h1:b3mkBEjcmxtft115cBIQk+2qz1HEb2ExDdduVQqN4 github.com/protolambda/zrnt v0.32.3/go.mod h1:A0fezkp9Tt3GBLATSPIbuY4ywYESyAuc/FFmPKg8Lqs= github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= @@ -972,6 +994,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -1313,6 +1336,7 @@ golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1371,6 +1395,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1445,6 +1471,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/crawler/ethereum.go b/pkg/crawler/ethereum.go index 65904cc..c76c388 100644 --- a/pkg/crawler/ethereum.go +++ b/pkg/crawler/ethereum.go @@ -145,31 +145,12 @@ func NewEthereumCrawler(mainCtx *cli.Context, conf config.EthereumCrawlerConfig) // create a gossipsub routing gs := gossipsub.NewGossipSub(ctx, host.Host(), dbClient) - // generate a new subnets-handler - ethMsgHandler, err := eth.NewEthMessageHandler(ethNode.GetNetworkGenesis(), conf.ValPubkeys) + // subscribe to all the network topics + ethMsgHandler, err := subscribeToGossipSubTopics(gs) if err != nil { cancel() return nil, err } - // subscribe the topics - for _, top := range conf.GossipTopics { - var msgHandler gossipsub.MessageHandler - switch top { - case eth.BeaconBlockTopicBase: - msgHandler = ethMsgHandler.BeaconBlockMessageHandler - default: - log.Error("untraceable gossipsub topic", top) - continue - - } - topic := eth.ComposeTopic(conf.ForkDigest, top) - gs.JoinAndSubscribe(topic, msgHandler, conf.PersistMsgs) - } - // subcribe to attestation subnets - for _, subnet := range conf.Subnets { - subTopics := eth.ComposeAttnetsTopic(conf.ForkDigest, subnet) - gs.JoinAndSubscribe(subTopics, ethMsgHandler.SubnetMessageHandler, conf.PersistMsgs) - } // generate the peering strategy pStrategy, err := peering.NewPruningStrategy( @@ -262,3 +243,67 @@ func (c *EthereumCrawler) Close() { c.Events.Stop() c.cancel() } + + +func subscribeToGossipSubTopics(gs *gossipsub.GossipSub) (*EthMessageHandler, error) { + + // generate a new subnets-handler + ethMsgHandler, err := eth.NewEthMessageHandler(ethNode.GetNetworkGenesis(), conf.ValPubkeys) + if err != nil { + return nil, err + } + + emptyhandler := func(msg *pubsub.Message) (PersistableMsg, error) { + return &DummyMessage, nil + } + + // subscribe to all topics + for _, top := range ethereum.EthereumValidTopics { + var msgHandler gossipsub.MessageHandler + switch top { + case eth.BeaconBlockTopicBase: + msgHandler = ethMsgHandler.BeaconBlockMessageHandler + case eth.BeaconAggregationAndProofMessageHandler: + msgHandler = ethMsgHandler.BeaconAggregationAndProofMessageHandler + case eth.AttestationSubnetsTopicBase: + for _, subnet := range conf.AttestationSubnetLimit { + t := eth.ComposeSubnetTopic(eth.AttestationSubnetsTopicBase, conf.ForkDigest, subnet) + gs.JoinAndSubscribe(t, ethMsgHandler.SubnetMessageHandler, conf.PersistMsgs) + } + case eth.VoluntaryExitTopicBase: + + case eth.ProposerSlashingTopicBase: + + case eth.AttesterSlashingTopicBase: + + case eth.SyncCommitteeAggregationsTopicBase: + + case eth.SyncCommitteeSubnetsTopicBase: + for _, subnet := range conf.SyncCommitteeSubnetLimit { + t := eth.ComposeSubnetTopic(eth.SyncCommitteeSubnetsTopicBase, conf.ForkDigest, subnet) + gs.JoinAndSubscribe(t, ethMsgHandler.SubnetMessageHandler, conf.PersistMsgs) + } + case eth.BLStoExectionChangeTopicBase: + + case eth.BlobsSubnetsTopicBase: + for _, subnet := range conf.BlobSubnetLimit { + t := eth.ComposeSubnetTopic(eth.BlobsSubnetsTopicBase, conf.ForkDigest, subnet) + gs.JoinAndSubscribe(t, ethMsgHandler.SubnetMessageHandler, conf.PersistMsgs) + } + + default: + log.Error("untraceable gossipsub topic", top) + continue + } + topic := eth.ComposeTopic(conf.ForkDigest, top) + gs.JoinAndSubscribe(topic, msgHandler, conf.PersistMsgs) + } + // subcribe to attestation subnets + for _, subnet := range conf.Subnets { + subTopics := eth.ComposeAttnetsTopic(conf.ForkDigest, subnet) + gs.JoinAndSubscribe(subTopics, ethMsgHandler.SubnetMessageHandler, conf.PersistMsgs) + } + + return ethMsgHandler, nil +} + diff --git a/pkg/networks/ethereum/gossip_handlers.go b/pkg/networks/ethereum/gossip_handlers.go index 00ac976..fb5b576 100644 --- a/pkg/networks/ethereum/gossip_handlers.go +++ b/pkg/networks/ethereum/gossip_handlers.go @@ -3,21 +3,23 @@ package ethereum import ( "bytes" "fmt" + "sync" + "time" + "github.com/protolambda/zrnt/eth2/beacon/altair" "github.com/protolambda/zrnt/eth2/beacon/deneb" "github.com/protolambda/zrnt/eth2/beacon/phase0" - "sync" - "time" + "github.com/protolambda/zrnt/eth2/beacon/common" + "github.com/protolambda/zrnt/eth2/configs" + "github.com/protolambda/ztyp/codec" + attdeneb "github.com/attestantio/go-eth2-client/spec/deneb" // bls "github.com/phoreproject/github.com/bls/g1pubs" + pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/migalabs/armiarma/pkg/gossipsub" - "github.com/protolambda/zrnt/eth2/beacon/common" - "github.com/protolambda/zrnt/eth2/configs" - "github.com/protolambda/ztyp/codec" "github.com/golang/snappy" - pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -37,6 +39,7 @@ type EthMessageHandler struct { genesisTime time.Time pubkeys []*common.BLSPubkey // pubkeys of those validators we want to track + // SubHanndlers for each gossipsub topic (Oriented mostly for HTTP SSE) callbackM sync.RWMutex messageCallbacks map[EthereumGossipTopic][]func(event interface{}) } @@ -321,6 +324,7 @@ func (mh *EthMessageHandler) TrackedSyncAggregate(msg *pubsub.Message) (gossipsu } trackedSyncAggregate := &TrackedSyncAggregate{ TrackedMessage: TrackedMessage{ + Msg: syncContribution, MsgID: msg.ID, Time: msg.ArrivalTime, Sender: msg.ReceivedFrom, @@ -350,21 +354,22 @@ func (mh *EthMessageHandler) TrackedSyncVotes(msg *pubsub.Message) (gossipsub.Pe return nil, err } msgBuf := bytes.NewBuffer(msgBytes) - syncContribution := new(altair.SyncCommitteeMessage) + syncVote := new(altair.SyncCommitteeMessage) - err = syncContribution.Deserialize(codec.NewDecodingReader(msgBuf, uint64(len(msgBuf.Bytes())))) + err = syncVote.Deserialize(codec.NewDecodingReader(msgBuf, uint64(len(msgBuf.Bytes())))) if err != nil { return nil, err } trackedSyncMsg := &TrackedSyncMessage{ TrackedMessage: TrackedMessage{ + Msg: syncVote, MsgID: msg.ID, Time: msg.ArrivalTime, Sender: msg.ReceivedFrom, }, - ValIndex: uint64(syncContribution.ValidatorIndex), - TimeInSlot: GetTimeInSlot(mh.genesisTime, msg.ArrivalTime, int64(syncContribution.Slot)), - Slot: uint64(syncContribution.Slot), + ValIndex: uint64(syncVote.ValidatorIndex), + TimeInSlot: GetTimeInSlot(mh.genesisTime, msg.ArrivalTime, int64(syncVote.Slot)), + Slot: uint64(syncVote.Slot), } // check if there is any callback callbacks, ok := mh.getCallBacks(BeaconSubnetSyncCommitteeVoteTopic) @@ -375,3 +380,42 @@ func (mh *EthMessageHandler) TrackedSyncVotes(msg *pubsub.Message) (gossipsub.Pe } return trackedSyncMsg, nil } + + +func (mh *EthMessageHandler) TrackedBlobSidecars(msg *pubsub.Message) (gossipsub.PersistableMsg, error) { + t := time.Now() + defer log.Trace("total time to handle msg:", time.Since(t)) + topic := *msg.Topic + + // extract the data from the raw message + msgBytes, err := EthMessageBaseHandler(topic, msg) + if err != nil { + return nil, err + } + msgBuf := bytes.NewBuffer(msgBytes) + blobSidecar := new(attdeneb.BlobSidecar) + + err = blobSidecar.UnmarshalSSZ(msgBuf.Bytes()) + if err != nil { + return nil, err + } + trackedSyncMsg := &TrackedBlobSidecards{ + TrackedMessage: TrackedMessage{ + Msg: blobSidecar, + MsgID: msg.ID, + Time: msg.ArrivalTime, + Sender: msg.ReceivedFrom, + }, + BlobIndex: uint64(blobSidecar.Index), + BeaconBlockRoot: blobSidecar.SignedBlockHeader.Message.Root.String(), + + } + // check if there is any callback + callbacks, ok := mh.getCallBacks(BeaconSubnetSyncCommitteeVoteTopic) + if ok { + for _, callback := range callbacks { + callback(trackedSyncMsg) // TODO: update to submite the event + } + } + return trackedSyncMsg, nil +} \ No newline at end of file diff --git a/pkg/networks/ethereum/gossip_messages.go b/pkg/networks/ethereum/gossip_messages.go index 9cefd23..fe7401e 100644 --- a/pkg/networks/ethereum/gossip_messages.go +++ b/pkg/networks/ethereum/gossip_messages.go @@ -39,10 +39,20 @@ func (m *TrackedMessage) ArrivalTime() peer.ID { return m.Sender } -func (m *TrackedMessage) Message() any { +func (m *TrackedMessage) Message() any { // any = Marshaleable return m.Msg } + +// Dummy message (For not yet ready topics) +type DummyMessage struct { + TrackedMessage +} + +func (m *DummyMessage) IsZero() bool { + return true +} + // Ethereum Message-Specifics // Beacon Block type TrackedBeaconBlock struct { @@ -140,6 +150,13 @@ func (a *TrackedSyncMessage) IsZero() bool { // BLS_Changes (TODO) // blobs (TODO: - zrnt doesn't include the blob struct, still looking for the time to implement the entire structure, the SSZ serialization, the view, the tree hashing, etc) +// Experimental using the Eth2Clients library from attestant -> https://github.com/attestantio/go-eth2-client/blob/2d68bcd60d23ca11bbf073332f86a15b83b7a265/spec/deneb/blobsidecar.go#L24 type TrackedBlobSidecards struct { TrackedMessage + BlobIndex uint64 + BeaconBlockRoot string } + +func (a *TrackedBlobSidecards) IsZero() bool { + return a.BeaconBlockRoot != "" +} \ No newline at end of file diff --git a/pkg/networks/ethereum/gossip_topics.go b/pkg/networks/ethereum/gossip_topics.go index 1f10a23..319765b 100644 --- a/pkg/networks/ethereum/gossip_topics.go +++ b/pkg/networks/ethereum/gossip_topics.go @@ -31,17 +31,32 @@ const ( ProposerSlashingTopicBase string = "proposer_slashing" AttesterSlashingTopicBase string = "attester_slashing" AttestationSubnetsTopicBase string = "beacon_attestation_{__subnet_id__}" - SubnetLimit = 64 + AttestationSubnetLimit = 64 SyncCommitteeAggregationsTopicBase string = "sync_committee_contribution_and_proof" SyncCommitteeSubnetsTopicBase string = "sync_committee_{__subnet_id__}" - SyncCommitteeLimit = 4 + SyncCommitteeSubnetLimit = 4 BLStoExectionChangeTopicBase string = "bls_to_execution_change" BlobsSubnetsTopicBase string = "blob_sidecar_{__subnet_id__}" + BlobSubnetLimit = 6 // encoding-compression Encoding string = "ssz_snappy" ) +// valid ethereum topics +var EthereumValidTopics = []string{ + BeaconBlockTopicBase, + BeaconAggregateAndProofTopicBase, + AttestationSubnetsTopicBase, + VoluntaryExitTopicBase, + ProposerSlashingTopicBase, + AttesterSlashingTopicBase, + SyncCommitteeAggregationsTopicBase, + SyncCommitteeSubnetsTopicBase, + BLStoExectionChangeTopicBase, + BlobsSubnetsTopicBase, +} + // EthTopicPretty returns the topic based on its message type in a pretty version of it. // It would return "beacon_block" out of the given "/eth2/b5303f2a/beacon_block/ssz_snappy" topic func EthTopicPretty(eth2topic string) string { @@ -60,7 +75,7 @@ func ComposeTopic(forkDigest string, messageTypeName string) string { // ComposeSubnetTopic generates the GossipSub topic for the given ForkDigest, base, and subnet func ComposeSubnetTopic(base, forkDigest string, subnet int) string { - if subnet > SubnetLimit || subnet <= 0 { + if subnet <= 0 { return "" }