fix data update by making it concurrent

Makefile

@@ -60,7 +60,7 @@ update-bids-website:
 	go run . service bidcollect --build-website --build-website-upload
 
 dev-website:
-	go run . service website --dev
+	DB_DONT_APPLY_SCHEMA=1 go run . service website --dev
 
 dev-bids-website:
 	go run . service bidcollect --devserver

README.md

@@ -20,6 +20,10 @@ Additional URLs:
   - https://www.relayscan.io/stats/day/2023-06-20
   - https://www.relayscan.io/stats/day/2023-06-20/json
 
+**Bid Archive**
+
+https://bidarchive.relayscan.io
+
 ## Notes
 
 - Work in progress

scripts/send-pushover-notification.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+if [ -z "$1" ]; then
+  echo "Usage: $0 <message>"
+  exit 1
+fi
+
+curl -s \
+        --form-string "token=$PUSHOVER_APP_TOKEN" \
+        --form-string "user=$PUSHOVER_APP_KEY" \
+        --form-string "message=$1" \
+        https://api.pushover.net/1/messages.json

services/website/html.go

@@ -13,6 +13,8 @@ type Stats struct {
 	Since time.Time
 	Until time.Time
 
+	TimeStr string // i.e. 24h, 12h, 1h or 7d
+
 	TopRelays          []*database.TopRelayEntry
 	TopBuilders        []*database.TopBuilderEntry
 	BuilderProfits     []*database.BuilderProfitEntry
@@ -29,17 +31,16 @@ func NewStats() *Stats {
 }
 
 type HTMLData struct {
-	Title string
+	Title     string
+	TimeSpans []string
+	TimeSpan  string
+	View      string // overview or builder-profit
 
-	GeneratedAt        time.Time
-	LastDataTime       time.Time
-	LastDataTimeString string
-	LastUpdateSlot     uint64
-	TimeSpans          []string
+	Stats *Stats // stats for this view
 
-	TimeSpan string
-	View     string
-	Stats    *Stats
+	LastUpdateSlot    uint64
+	LastUpdateTime    time.Time
+	LastUpdateTimeStr string
 }
 
 type HTMLDataDailyStats struct {

services/website/templates/base.html

@@ -44,6 +44,7 @@
             <a class="pure-menu-heading" href="/"><img src="/static/favicon/apple-touch-icon.png" height="16" /> relayscan.io</a>
 
             <ul class="pure-menu-list">
+                <li class="pure-menu-item"><a href="https://bidarchive.relayscan.io/" class="pure-menu-link">Bid Archive</a></li>
                 <li class="pure-menu-item"><a href="https://github.com/flashbots/relayscan" class="pure-menu-link">About</a></li>
                 <li class="pure-menu-item"><a href="https://twitter.com/relayscan_io" class="pure-menu-link"><i class="bi bi-twitter"></i></a></li>
             </ul>

services/website/templates/index.html

@@ -1,5 +1,5 @@
 {{ define "content" }}
-{{ $lastDataTime := .LastDataTimeString }}
+{{ $lastDataTime := .LastUpdateTime }}
 {{ $time := .TimeSpan }}
 {{ $view := .View }}
 
@@ -9,8 +9,8 @@ <h1 style="margin-bottom:10px;">MEV-Boost Analytics</h1>
 
         <p style="color: #6d6d6d; margin-top:0; line-height: 1.4em;">
             <small>
-                Updated {{ .LastDataTimeString }} UTC / Slot {{ .LastUpdateSlot }}<br>
-                <span id="updated_ago">({{ .LastDataTime | humanTime }})</span>
+                Updated at slot {{ .LastUpdateSlot }}
+                <span id="updated_ago">({{ $lastDataTime | humanTime }})</span>
             </small>
         </p>
         <p id="view-type">
@@ -207,7 +207,7 @@ <h1 style="margin-bottom:10px;">MEV-Boost Analytics</h1>
         return sec + " seconds";
     }
 
-    var referenceTime = new Date("{{ .LastDataTime }}");
+    var referenceTime = new Date("{{ $lastDataTime }}");
     var timeAgo = timeSince(referenceTime);
     if (timeAgo != "") {
         document.getElementById("updated_ago").innerText = "(" + timeAgo + " ago)";

services/website/types.go

@@ -1,12 +1,5 @@
 package website
 
-// type statsResp struct {
-// 	GeneratedAt uint64                      `json:"generated_at"`
-// 	DataStartAt uint64                      `json:"data_start_at"`
-// 	TopRelays   []*database.TopRelayEntry   `json:"top_relays"`
-// 	TopBuilders []*database.TopBuilderEntry `json:"top_builders"`
-// }
-
 type HTTPErrorResp struct {
 	Code    int    `json:"code"`
 	Message string `json:"message"`