@@ -2,10 +2,12 @@ package cmd
22
33import (
44 "bytes"
5+ "context"
56 "encoding/json"
67 "errors"
78 "fmt"
89 "io"
10+ "math"
911 "net/http"
1012 "os"
1113 "path/filepath"
@@ -20,6 +22,7 @@ import (
2022
2123 "github.com/smartcontractkit/chainlink/v2/core/sessions"
2224 "github.com/smartcontractkit/chainlink/v2/core/utils"
25+ "github.com/smartcontractkit/chainlink/v2/core/web"
2326 "github.com/smartcontractkit/chainlink/v2/core/web/presenters"
2427)
2528
@@ -65,6 +68,10 @@ func initAdminSubCmds(s *Shell) []cli.Command {
6568 Usage : "output directory of the captured profile" ,
6669 Value : "/tmp/" ,
6770 },
71+ cli.StringSliceFlag {
72+ Name : "vitals, v" ,
73+ Usage : "vitals to collect, can be specified multiple times. Options: 'allocs', 'block', 'cmdline', 'goroutine', 'heap', 'mutex', 'profile', 'threadcreate', 'trace'" ,
74+ },
6875 },
6976 },
7077 {
@@ -324,30 +331,114 @@ func (s *Shell) Profile(c *cli.Context) error {
324331
325332 genDir := filepath .Join (baseDir , "debuginfo-" + time .Now ().Format (time .RFC3339 ))
326333
327- if err := os .Mkdir (genDir , 0o755 ); err != nil {
334+ vitals := c .StringSlice ("vitals" )
335+ if len (vitals ) == 0 {
336+ vitals = []string {
337+ "allocs" , // A sampling of all past memory allocations
338+ "block" , // Stack traces that led to blocking on synchronization primitives
339+ "cmdline" , // The command line invocation of the current program
340+ "goroutine" , // Stack traces of all current goroutines
341+ "heap" , // A sampling of memory allocations of live objects.
342+ "mutex" , // Stack traces of holders of contended mutexes
343+ "profile" , // CPU profile.
344+ "threadcreate" , // Stack traces that led to the creation of new OS threads
345+ "trace" , // A trace of execution of the current program.
346+ }
347+ }
348+
349+ plugins , err := s .discoverPlugins (ctx )
350+ if err != nil {
328351 return s .errorOut (err )
329352 }
330- var wgPprof sync. WaitGroup
331- vitals := [] string {
332- "allocs" , // A sampling of all past memory allocations
333- "block" , // Stack traces that led to blocking on synchronization primitives
334- "cmdline" , // The command line invocation of the current program
335- "goroutine" , // Stack traces of all current goroutines
336- "heap" , // A sampling of memory allocations of live objects.
337- "mutex" , // Stack traces of holders of contended mutexes
338- "profile " // CPU profile.
339- "threadcreate" , // Stack traces that led to the creation of new OS threads
340- "trace" , // A trace of execution of the current program.
353+ var names [] string
354+ for _ , group := range plugins {
355+ if name := group . Labels [ web . LabelMetaPluginName ]; name != "" {
356+ names = append ( names , name )
357+ }
358+ }
359+
360+ if len ( names ) == 0 {
361+ s . Logger . Infof ( "Collecting profiles: %v "vitals )
362+ } else {
363+ s . Logger . Infof ( "Collecting profiles from host and %d plugins: %v" , len ( names ), vitals )
341364 }
342- wgPprof .Add (len (vitals ))
343- s .Logger .Infof ("Collecting profiles: %v" , vitals )
344365 s .Logger .Infof ("writing debug info to %s" , genDir )
345366
367+ var wg sync.WaitGroup
368+ errs := make ([]error , len (names )+ 1 )
369+ wg .Add (len (names ) + 1 )
370+ go func () {
371+ defer wg .Done ()
372+ errs [0 ] = s .profile (ctx , genDir , "" , vitals , seconds )
373+ }()
374+ for i , name := range names {
375+ go func () {
376+ defer wg .Done ()
377+ errs [i ] = s .profile (ctx , genDir , name , vitals , seconds )
378+ }()
379+ }
380+ wg .Wait ()
381+
382+ err = errors .Join (errs ... )
383+ if err != nil {
384+ return s .errorOut (err )
385+ }
386+ return nil
387+ }
388+ func (s * Shell ) discoverPlugins (ctx context.Context ) (
389+ got []struct {
390+ Targets []string `yaml:"targets"`
391+ Labels map [string ]string `yaml:"labels"`
392+ },
393+ err error ,
394+ ) {
395+ resp , err := s .HTTP .Get (ctx , "/discovery" )
396+ if err != nil {
397+ return
398+ }
399+ defer func () {
400+ if resp .Body != nil {
401+ resp .Body .Close ()
402+ }
403+ }()
404+ data , err := io .ReadAll (resp .Body )
405+ if err != nil {
406+ return
407+ }
408+
409+ if err = json .Unmarshal (data , & got ); err != nil {
410+ s .Logger .Errorf ("failed to unmarshal discovery response: %s" , string (data ))
411+ return
412+ }
413+ return
414+ }
415+
416+ func (s * Shell ) profile (ctx context.Context , genDir string , name string , vitals []string , seconds uint ) error {
417+ lggr := s .Logger
418+ path := "/v2"
419+ if name != "" {
420+ genDir = filepath .Join (genDir , "plugins" , name )
421+ path += "/plugins/" + name
422+ lggr = lggr .With ("plugin" , name )
423+ }
424+ if err := os .MkdirAll (genDir , 0o755 ); err != nil {
425+ return fmt .Errorf ("failed to create directory: %w" , err )
426+ }
427+
428+ timeout := seconds + max (10 , seconds >> 2 ) // +25%
429+ if timeout > math .MaxInt64 {
430+ return fmt .Errorf ("profile timeout %d seconds overflows max int64: %d" , seconds , math .MaxInt64 )
431+ }
432+
346433 errs := make (chan error , len (vitals ))
434+ var wgPprof sync.WaitGroup
435+ wgPprof .Add (len (vitals ))
347436 for _ , vt := range vitals {
348- go func (vt string ) {
437+ go func (ctx context. Context , vt string ) {
349438 defer wgPprof .Done ()
350- uri := fmt .Sprintf ("/v2/debug/pprof/%s?seconds=%d" , vt , seconds )
439+ ctx , cancel := context .WithTimeout (ctx , time .Duration (timeout )* time .Second ) //nolint:gosec // G115 false positive
440+ defer cancel ()
441+ uri := fmt .Sprintf (path + "/debug/pprof/%s?seconds=%d" , vt , seconds )
351442 resp , err := s .HTTP .Get (ctx , uri )
352443 if err != nil {
353444 errs <- fmt .Errorf ("error collecting %s: %w" , vt , err )
@@ -403,12 +494,12 @@ func (s *Shell) Profile(c *cli.Context) error {
403494 errs <- fmt .Errorf ("error closing file for %s: %w" , vt , err )
404495 return
405496 }
406- }(vt )
497+ }(ctx , vt )
407498 }
408499 wgPprof .Wait ()
409500 close (errs )
410- // Atmost one err is emitted per vital.
411- s . Logger .Infof ("collected %d/%d profiles" , len (vitals )- len (errs ), len (vitals ))
501+ // At most one err is emitted per vital.
502+ lggr .Infof ("collected %d/%d profiles" , len (vitals )- len (errs ), len (vitals ))
412503 if len (errs ) > 0 {
413504 var merr error
414505 for err := range errs {
0 commit comments