From 6b25e22e4f9656b4d398c759398b8bf1a08410b1 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Mon, 19 Aug 2024 09:26:26 +0100 Subject: [PATCH 1/2] added heap profile --- api/api_handler.go | 24 +++++++++++ internal/erigon_node/erigon_client.go | 2 + internal/erigon_node/profile.go | 62 +++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 internal/erigon_node/profile.go diff --git a/api/api_handler.go b/api/api_handler.go index f4b3e76..e0a0415 100644 --- a/api/api_handler.go +++ b/api/api_handler.go @@ -1,6 +1,8 @@ package api import ( + "bytes" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -271,6 +273,27 @@ func (h *APIHandler) UniversalRequest(w http.ResponseWriter, r *http.Request) { w.Write(jsonData) } +func (h *APIHandler) HeapProfile(w http.ResponseWriter, r *http.Request) { + client, err := h.findNodeClient(r) + + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + heap, err := client.FindHeapProfile(r.Context()) + + if err != nil { + http.Error(w, fmt.Sprintf("Unable to fetch sync stage progress: %v", err), http.StatusInternalServerError) + return + } + + encoded := base64.StdEncoding.EncodeToString(heap) + + //w.Header().Set("Content-Type", "text/plain") + w.Write(bytes.NewBufferString(encoded).Bytes()) +} + func NewAPIHandler( sessions sessions.CacheService, erigonNode erigon_node.Client, @@ -291,6 +314,7 @@ func NewAPIHandler( r.Get("/sessions/{sessionId}/nodes/{nodeId}/headers/download-summary", r.HeadersDownload) r.Get("/sessions/{sessionId}/nodes/{nodeId}/sync-stages", r.SyncStages) r.Get("/v2/sessions/{sessionId}/nodes/{nodeId}/*", r.UniversalRequest) + r.Get("/sessions/{sessionId}/nodes/{nodeId}/profile/heap", r.HeapProfile) return r } diff --git a/internal/erigon_node/erigon_client.go b/internal/erigon_node/erigon_client.go index a06eafe..d0e2794 100644 --- a/internal/erigon_node/erigon_client.go +++ b/internal/erigon_node/erigon_client.go @@ -93,5 +93,7 @@ type Client interface { BodiesDownload(ctx context.Context, w http.ResponseWriter) HeadersDownload(ctx context.Context, w http.ResponseWriter) + FindHeapProfile(ctx context.Context) ([]byte, error) + fetch(ctx context.Context, method string, params url.Values) (*NodeRequest, error) } diff --git a/internal/erigon_node/profile.go b/internal/erigon_node/profile.go new file mode 100644 index 0000000..b33460b --- /dev/null +++ b/internal/erigon_node/profile.go @@ -0,0 +1,62 @@ +package erigon_node + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" +) + +type ProfileContent struct { + Chunk []byte `json:"chunk"` +} + +func (c *NodeClient) FindHeapProfile(ctx context.Context) ([]byte, error) { + request, err := c.fetch(ctx, "heap-profile", nil) + + if err != nil { + return nil, err + } + + _, result, err := request.nextResult(ctx) + + if err != nil { + return nil, err + } + + var content ProfileContent + + if err := json.Unmarshal(result, &content); err != nil { + return nil, err + } + + //result is a file content so I need to save it to file and return the file path + tempFile, err := os.CreateTemp("", "heap-profile-*.pprof") + if err != nil { + return nil, err + } + + defer func() { + if err := tempFile.Close(); err != nil { + fmt.Printf("Error closing temporary file: %v\n", err) + } + + // Remove the file after closing it + if err := os.Remove(tempFile.Name()); err != nil { + fmt.Printf("Error removing temporary file: %v\n", err) + } + }() + + if _, err := tempFile.Write(content.Chunk); err != nil { + return nil, err + } + + cmd := exec.Command("go", "tool", "pprof", "-png", tempFile.Name()) + svgOutput, err := cmd.Output() + if err != nil { + return nil, err + } + + return svgOutput, nil +} From 18b65d2600cffe4ba2fb39828ddaf07db8e14d65 Mon Sep 17 00:00:00 2001 From: Dmytro Vovk Date: Thu, 5 Sep 2024 18:09:28 +0100 Subject: [PATCH 2/2] added support to query all pprofs --- api/api_handler.go | 51 ++++++++++++++------------- internal/erigon_node/erigon_client.go | 2 +- internal/erigon_node/profile.go | 6 ++-- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/api/api_handler.go b/api/api_handler.go index e0a0415..2c763a4 100644 --- a/api/api_handler.go +++ b/api/api_handler.go @@ -2,12 +2,14 @@ package api import ( "bytes" + "context" "encoding/base64" "encoding/json" "fmt" "net/http" "path" "strconv" + "strings" "github.com/go-chi/chi/v5" @@ -247,8 +249,6 @@ func (h *APIHandler) findNodeClient(r *http.Request) (erigon_node.Client, error) } func (h *APIHandler) UniversalRequest(w http.ResponseWriter, r *http.Request) { - apiStr := chi.URLParam(r, "*") - client, err := h.findNodeClient(r) if err != nil { @@ -256,42 +256,44 @@ func (h *APIHandler) UniversalRequest(w http.ResponseWriter, r *http.Request) { return } - response, err := client.GetResponse(r.Context(), apiStr) + pprof, data, err := GetResponseData(r.Context(), client, chi.URLParam(r, "*")) if err != nil { http.Error(w, fmt.Sprintf("Unable to fetch snapshot files list: %v", err), http.StatusInternalServerError) return } - jsonData, err := json.Marshal(response) - - if err != nil { - api_internal.EncodeError(w, r, err) + if pprof { + encoded := base64.StdEncoding.EncodeToString(data) + data = bytes.NewBufferString(encoded).Bytes() + } else { + w.Header().Set("Content-Type", "application/json") } - w.Header().Set("Content-Type", "application/json") - w.Write(jsonData) + w.Write(data) } -func (h *APIHandler) HeapProfile(w http.ResponseWriter, r *http.Request) { - client, err := h.findNodeClient(r) +func GetResponseData(ctx context.Context, client erigon_node.Client, request string) (bool, []byte, error) { + if strings.Contains(request, "pprof") { + data, err := client.FindProfile(ctx, request) + if err != nil { + return true, nil, err + } - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } + return true, data, nil + } else { + response, err := client.GetResponse(ctx, request) + if err != nil { + return false, nil, err + } - heap, err := client.FindHeapProfile(r.Context()) + jsonData, err := json.Marshal(response) + if err != nil { + return false, nil, err + } - if err != nil { - http.Error(w, fmt.Sprintf("Unable to fetch sync stage progress: %v", err), http.StatusInternalServerError) - return + return false, jsonData, nil } - - encoded := base64.StdEncoding.EncodeToString(heap) - - //w.Header().Set("Content-Type", "text/plain") - w.Write(bytes.NewBufferString(encoded).Bytes()) } func NewAPIHandler( @@ -314,7 +316,6 @@ func NewAPIHandler( r.Get("/sessions/{sessionId}/nodes/{nodeId}/headers/download-summary", r.HeadersDownload) r.Get("/sessions/{sessionId}/nodes/{nodeId}/sync-stages", r.SyncStages) r.Get("/v2/sessions/{sessionId}/nodes/{nodeId}/*", r.UniversalRequest) - r.Get("/sessions/{sessionId}/nodes/{nodeId}/profile/heap", r.HeapProfile) return r } diff --git a/internal/erigon_node/erigon_client.go b/internal/erigon_node/erigon_client.go index d0e2794..1532b90 100644 --- a/internal/erigon_node/erigon_client.go +++ b/internal/erigon_node/erigon_client.go @@ -93,7 +93,7 @@ type Client interface { BodiesDownload(ctx context.Context, w http.ResponseWriter) HeadersDownload(ctx context.Context, w http.ResponseWriter) - FindHeapProfile(ctx context.Context) ([]byte, error) + FindProfile(ctx context.Context, profile string) ([]byte, error) fetch(ctx context.Context, method string, params url.Values) (*NodeRequest, error) } diff --git a/internal/erigon_node/profile.go b/internal/erigon_node/profile.go index b33460b..60ecbea 100644 --- a/internal/erigon_node/profile.go +++ b/internal/erigon_node/profile.go @@ -12,8 +12,8 @@ type ProfileContent struct { Chunk []byte `json:"chunk"` } -func (c *NodeClient) FindHeapProfile(ctx context.Context) ([]byte, error) { - request, err := c.fetch(ctx, "heap-profile", nil) +func (c *NodeClient) FindProfile(ctx context.Context, profile string) ([]byte, error) { + request, err := c.fetch(ctx, profile, nil) if err != nil { return nil, err @@ -32,7 +32,7 @@ func (c *NodeClient) FindHeapProfile(ctx context.Context) ([]byte, error) { } //result is a file content so I need to save it to file and return the file path - tempFile, err := os.CreateTemp("", "heap-profile-*.pprof") + tempFile, err := os.CreateTemp("", "profile-*.pprof") if err != nil { return nil, err }