From eb0bb8797491a3e73a0d9abc369683fd36e3a6b6 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Fri, 23 Feb 2024 14:04:10 +0100 Subject: [PATCH 1/2] added index, more logging for loading data --- README.md | 7 +++++++ database/schema.go | 1 + services/website/html.go | 9 +++++++++ services/website/webserver.go | 30 +++++++++++++++++++++++------- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 47cc7ae..27502a1 100644 --- a/README.md +++ b/README.md @@ -102,3 +102,10 @@ make test make test-race make build ``` + + +### Updating relayscan + +Notes for updating relayscan: + +- Relay payloads are selected by `inserted_at`. When adding a new relay, you probably want to manually subtract a day from `inserted_at` so they don't show up all for today (`UPDATE mainnet_data_api_payload_delivered SET inserted_at = inserted_at - INTERVAL '1 DAY' WHERE relay='newrelay.xyz';`). See also https://github.com/flashbots/relayscan/issues/28 diff --git a/database/schema.go b/database/schema.go index 11327d1..be0e6bd 100644 --- a/database/schema.go +++ b/database/schema.go @@ -97,6 +97,7 @@ CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_value_wei_idx ON CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_valuecheck_ok_idx ON ` + TableDataAPIPayloadDelivered + `("value_check_ok"); CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_slotmissed_idx ON ` + TableDataAPIPayloadDelivered + `("slot_missed"); CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_cb_diff_eth_idx ON ` + TableDataAPIPayloadDelivered + `("coinbase_diff_eth"); +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_insertedat_relay_idx ON ` + TableDataAPIPayloadDelivered + `("inserted_at", "relay"); CREATE TABLE IF NOT EXISTS ` + TableDataAPIBuilderBid + ` ( diff --git a/services/website/html.go b/services/website/html.go index 7daa2c2..9e7c333 100644 --- a/services/website/html.go +++ b/services/website/html.go @@ -18,6 +18,15 @@ type Stats struct { TopBuildersByRelay map[string][]*database.TopBuilderEntry } +func NewStats() *Stats { + return &Stats{ + TopRelays: make([]*database.TopRelayEntry, 0), + TopBuilders: make([]*database.TopBuilderEntry, 0), + BuilderProfits: make([]*database.BuilderProfitEntry, 0), + TopBuildersByRelay: make(map[string][]*database.TopBuilderEntry), + } +} + type HTMLData struct { Title string diff --git a/services/website/webserver.go b/services/website/webserver.go index ff37658..bf7a5c1 100644 --- a/services/website/webserver.go +++ b/services/website/webserver.go @@ -152,20 +152,26 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er now := time.Now().UTC() since := now.Add(-1 * duration.Abs()) + startTime := time.Now() topRelays, err := srv.db.GetTopRelays(since, now) if err != nil { return nil, err } + srv.log.WithField("duration", time.Since(startTime).String()).Debug("got top relays") + startTime = time.Now() topBuilders, err := srv.db.GetTopBuilders(since, now, "") if err != nil { return nil, err } + srv.log.WithField("duration", time.Since(startTime).String()).Debug("got top builders") + startTime = time.Now() builderProfits, err := srv.db.GetBuilderProfits(since, now) if err != nil { return nil, err } + srv.log.WithField("duration", time.Since(startTime).String()).Debug("got builder profits") stats = &Stats{ Since: since, @@ -178,6 +184,7 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er } // Query builders for each relay + startTime = time.Now() for _, relay := range topRelays { topBuildersForRelay, err := srv.db.GetTopBuilders(since, now, relay.Relay) if err != nil { @@ -185,6 +192,7 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er } stats.TopBuildersByRelay[relay.Relay] = consolidateBuilderEntries(topBuildersForRelay) } + srv.log.WithField("duration", time.Since(startTime).String()).Debug("got builders per relay") return stats, nil } @@ -217,34 +225,42 @@ func (srv *Webserver) updateHTML() { htmlData.LastUpdateSlot = entry.Slot } + startUpdate := time.Now() srv.log.Info("updating 24h stats...") stats["24h"], err = srv.getStatsForHours(24 * time.Hour) if err != nil { srv.log.WithError(err).Error("Failed to get stats for 24h") return } + srv.log.WithField("duration", time.Since(startUpdate).String()).Info("updated 24h stats") if !srv.opts.Only24h { - srv.log.Info("updating 7d stats...") - stats["7d"], err = srv.getStatsForHours(7 * 24 * time.Hour) - if err != nil { - srv.log.WithError(err).Error("Failed to get stats for 24h") - return - } - + startUpdate = time.Now() srv.log.Info("updating 12h stats...") stats["12h"], err = srv.getStatsForHours(12 * time.Hour) if err != nil { srv.log.WithError(err).Error("Failed to get stats for 12h") return } + srv.log.WithField("duration", time.Since(startUpdate).String()).Info("updated 12h stats") + startUpdate = time.Now() srv.log.Info("updating 1h stats...") stats["1h"], err = srv.getStatsForHours(1 * time.Hour) if err != nil { srv.log.WithError(err).Error("Failed to get stats for 1h") return } + srv.log.WithField("duration", time.Since(startUpdate).String()).Info("updated 1h stats") + + startUpdate = time.Now() + srv.log.Info("updating 7d stats...") + stats["7d"], err = srv.getStatsForHours(7 * 24 * time.Hour) + if err != nil { + srv.log.WithError(err).Error("Failed to get stats for 24h") + return + } + srv.log.WithField("duration", time.Since(startUpdate).String()).Info("updated 7d stats") } // Save the html data From 46bea8742ba941e63eb36a96072c2cfeba7f782b Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Sat, 24 Feb 2024 14:03:09 +0100 Subject: [PATCH 2/2] more debug logging --- database/database.go | 6 +++--- services/website/webserver.go | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/database/database.go b/database/database.go index 61c4746..f9dbd2c 100644 --- a/database/database.go +++ b/database/database.go @@ -116,14 +116,14 @@ func (s *DatabaseService) GetDataAPILatestBid(relay string) (*DataAPIBuilderBidE } func (s *DatabaseService) GetTopRelays(since, until time.Time) (res []*TopRelayEntry, err error) { - query := `SELECT relay, count(relay) as payloads FROM mainnet_data_api_payload_delivered WHERE inserted_at > $1 AND inserted_at < $2 GROUP BY relay ORDER BY payloads DESC;` + query := `SELECT relay, count(relay) as payloads FROM ` + TableDataAPIPayloadDelivered + ` WHERE inserted_at > $1 AND inserted_at < $2 GROUP BY relay ORDER BY payloads DESC;` err = s.DB.Select(&res, query, since.UTC(), until.UTC()) return res, err } func (s *DatabaseService) GetTopBuilders(since, until time.Time, relay string) (res []*TopBuilderEntry, err error) { query := `SELECT extra_data, count(extra_data) as blocks FROM ( - SELECT distinct(slot), extra_data FROM mainnet_data_api_payload_delivered WHERE inserted_at > $1 AND inserted_at < $2` + SELECT distinct(slot), extra_data FROM ` + TableDataAPIPayloadDelivered + ` WHERE inserted_at > $1 AND inserted_at < $2` if relay != "" { query += ` AND relay = '` + relay + `'` } @@ -144,7 +144,7 @@ func (s *DatabaseService) GetBuilderProfits(since, until time.Time) (res []*Buil round(sum(CASE WHEN coinbase_diff_eth IS NOT NULL THEN coinbase_diff_eth ELSE 0 END), 4) as total_profit, round(abs(sum(CASE WHEN coinbase_diff_eth < 0 THEN coinbase_diff_eth ELSE 0 END)), 4) as total_subsidies FROM ( - SELECT distinct(slot), extra_data, coinbase_diff_eth FROM mainnet_data_api_payload_delivered WHERE inserted_at > $1 AND inserted_at < $2 + SELECT distinct(slot), extra_data, coinbase_diff_eth FROM ` + TableDataAPIPayloadDelivered + ` WHERE inserted_at > $1 AND inserted_at < $2 ) AS x GROUP BY extra_data ORDER BY total_profit DESC;` diff --git a/services/website/webserver.go b/services/website/webserver.go index bf7a5c1..5b02274 100644 --- a/services/website/webserver.go +++ b/services/website/webserver.go @@ -152,26 +152,29 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er now := time.Now().UTC() since := now.Add(-1 * duration.Abs()) + srv.log.Debug("- loading top relays...") startTime := time.Now() topRelays, err := srv.db.GetTopRelays(since, now) if err != nil { return nil, err } - srv.log.WithField("duration", time.Since(startTime).String()).Debug("got top relays") + srv.log.WithField("duration", time.Since(startTime).String()).Debug("- got top relays") + srv.log.Debug("- loading top builders...") startTime = time.Now() topBuilders, err := srv.db.GetTopBuilders(since, now, "") if err != nil { return nil, err } - srv.log.WithField("duration", time.Since(startTime).String()).Debug("got top builders") + srv.log.WithField("duration", time.Since(startTime).String()).Debug("- got top builders") + srv.log.Debug("- loading builder profits...") startTime = time.Now() builderProfits, err := srv.db.GetBuilderProfits(since, now) if err != nil { return nil, err } - srv.log.WithField("duration", time.Since(startTime).String()).Debug("got builder profits") + srv.log.WithField("duration", time.Since(startTime).String()).Debug("- got builder profits") stats = &Stats{ Since: since, @@ -184,6 +187,7 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er } // Query builders for each relay + srv.log.Debug("- loading builders per relay...") startTime = time.Now() for _, relay := range topRelays { topBuildersForRelay, err := srv.db.GetTopBuilders(since, now, relay.Relay) @@ -192,7 +196,7 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er } stats.TopBuildersByRelay[relay.Relay] = consolidateBuilderEntries(topBuildersForRelay) } - srv.log.WithField("duration", time.Since(startTime).String()).Debug("got builders per relay") + srv.log.WithField("duration", time.Since(startTime).String()).Debug("- got builders per relay") return stats, nil }