@@ -2,6 +2,7 @@ package cmd
22
33import (
44 "bytes"
5+ "context"
56 "encoding/json"
67 "errors"
78 "fmt"
@@ -17,6 +18,7 @@ import (
1718 "github.com/urfave/cli"
1819
1920 cutils "github.com/smartcontractkit/chainlink-common/pkg/utils"
21+ "github.com/smartcontractkit/chainlink/v2/core/web"
2022
2123 "github.com/smartcontractkit/chainlink/v2/core/sessions"
2224 "github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -65,6 +67,10 @@ func initAdminSubCmds(s *Shell) []cli.Command {
6567 Usage : "output directory of the captured profile" ,
6668 Value : "/tmp/" ,
6769 },
70+ cli.StringSliceFlag {
71+ Name : "vitals, v" ,
72+ Usage : "vitals to collect, can be specified multiple times. Options: 'allocs', 'block', 'cmdline', 'goroutine', 'heap', 'mutex', 'profile', 'threadcreate', 'trace'" ,
73+ },
6874 },
6975 },
7076 {
@@ -324,30 +330,112 @@ func (s *Shell) Profile(c *cli.Context) error {
324330
325331 genDir := filepath .Join (baseDir , "debuginfo-" + time .Now ().Format (time .RFC3339 ))
326332
327- if err := os .Mkdir (genDir , 0o755 ); err != nil {
333+ vitals := c .StringSlice ("vitals" )
334+ if len (vitals ) == 0 {
335+ vitals = []string {
336+ "allocs" , // A sampling of all past memory allocations
337+ "block" , // Stack traces that led to blocking on synchronization primitives
338+ "cmdline" , // The command line invocation of the current program
339+ "goroutine" , // Stack traces of all current goroutines
340+ "heap" , // A sampling of memory allocations of live objects.
341+ "mutex" , // Stack traces of holders of contended mutexes
342+ "profile" , // CPU profile.
343+ "threadcreate" , // Stack traces that led to the creation of new OS threads
344+ "trace" , // A trace of execution of the current program.
345+ }
346+ }
347+
348+ plugins , err := s .discoverPlugins (ctx )
349+ if err != nil {
328350 return s .errorOut (err )
329351 }
330- var wgPprof sync.WaitGroup
331- vitals := []string {
332- "allocs" , // A sampling of all past memory allocations
333- "block" , // Stack traces that led to blocking on synchronization primitives
334- "cmdline" , // The command line invocation of the current program
335- "goroutine" , // Stack traces of all current goroutines
336- "heap" , // A sampling of memory allocations of live objects.
337- "mutex" , // Stack traces of holders of contended mutexes
338- "profile" , // CPU profile.
339- "threadcreate" , // Stack traces that led to the creation of new OS threads
340- "trace" , // A trace of execution of the current program.
352+ var names []string
353+ for _ , group := range plugins {
354+ if name := group .Labels [web .LabelMetaPluginName ]; name != "" {
355+ names = append (names , name )
356+ }
357+ }
358+
359+ if len (names ) == 0 {
360+
361+ s .Logger .Infof ("Collecting profiles: %v" , vitals )
362+ } else {
363+ s .Logger .Infof ("Collecting profiles from host and %d plugins: %v" , len (names ), vitals )
341364 }
342- wgPprof .Add (len (vitals ))
343- s .Logger .Infof ("Collecting profiles: %v" , vitals )
344365 s .Logger .Infof ("writing debug info to %s" , genDir )
345366
367+ var wg sync.WaitGroup
368+ errs := make ([]error , len (names )+ 1 )
369+ wg .Add (len (names ) + 1 )
370+ go func () {
371+ defer wg .Done ()
372+ errs [0 ] = s .profile (ctx , genDir , "" , vitals , seconds )
373+ }()
374+ for i , name := range names {
375+ go func () {
376+ defer wg .Done ()
377+ errs [i ] = s .profile (ctx , genDir , name , vitals , seconds )
378+ }()
379+ }
380+ wg .Wait ()
381+
382+ err = errors .Join (errs ... )
383+ if err != nil {
384+ return s .errorOut (err )
385+ }
386+ return nil
387+ }
388+ func (s * Shell ) discoverPlugins (ctx context.Context ) (
389+ got []struct {
390+ Targets []string `yaml:"targets"`
391+ Labels map [string ]string `yaml:"labels"`
392+ },
393+ err error ,
394+ ) {
395+ resp , err := s .HTTP .Get (ctx , fmt .Sprintf ("/discovery" ))
396+ if err != nil {
397+ return
398+ }
399+ defer func () {
400+ if resp .Body != nil {
401+ resp .Body .Close ()
402+ }
403+ }()
404+ data , err := io .ReadAll (resp .Body )
405+ if err != nil {
406+ return
407+ }
408+
409+ if err = json .Unmarshal (data , & got ); err != nil {
410+ s .Logger .Errorf ("failed to unmarshal discovery response: %s" , string (data ))
411+ return
412+ }
413+ return
414+ }
415+
416+ func (s * Shell ) profile (ctx context.Context , genDir string , name string , vitals []string , seconds uint ) error {
417+ lggr := s .Logger
418+ path := "/v2"
419+ if name != "" {
420+ genDir = filepath .Join (genDir , "plugins" , name )
421+ path += "/plugins/" + name
422+ lggr = lggr .With ("plugin" , name )
423+ }
424+ if err := os .MkdirAll (genDir , 0o755 ); err != nil {
425+ return fmt .Errorf ("failed to create directory: %w" , err )
426+ }
427+
428+ timeout := seconds + max (10 , seconds >> 2 ) // +25%
429+
346430 errs := make (chan error , len (vitals ))
431+ var wgPprof sync.WaitGroup
432+ wgPprof .Add (len (vitals ))
347433 for _ , vt := range vitals {
348- go func (vt string ) {
434+ go func (ctx context. Context , vt string ) {
349435 defer wgPprof .Done ()
350- uri := fmt .Sprintf ("/v2/debug/pprof/%s?seconds=%d" , vt , seconds )
436+ ctx , cancel := context .WithTimeout (ctx , time .Duration (timeout )* time .Second )
437+ defer cancel ()
438+ uri := fmt .Sprintf (path + "/debug/pprof/%s?seconds=%d" , vt , seconds )
351439 resp , err := s .HTTP .Get (ctx , uri )
352440 if err != nil {
353441 errs <- fmt .Errorf ("error collecting %s: %w" , vt , err )
@@ -403,12 +491,12 @@ func (s *Shell) Profile(c *cli.Context) error {
403491 errs <- fmt .Errorf ("error closing file for %s: %w" , vt , err )
404492 return
405493 }
406- }(vt )
494+ }(ctx , vt )
407495 }
408496 wgPprof .Wait ()
409497 close (errs )
410- // Atmost one err is emitted per vital.
411- s . Logger .Infof ("collected %d/%d profiles" , len (vitals )- len (errs ), len (vitals ))
498+ // At most one err is emitted per vital.
499+ lggr .Infof ("collected %d/%d profiles" , len (vitals )- len (errs ), len (vitals ))
412500 if len (errs ) > 0 {
413501 var merr error
414502 for err := range errs {
0 commit comments