diff --git a/api/server/server.go b/api/server/server.go index bd54d3481..435d8f87f 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "syscall" + "unicode" "github.com/fnproject/fn/api/agent" "github.com/fnproject/fn/api/agent/hybrid" @@ -63,6 +64,8 @@ const ( EnvCertKey = "FN_NODE_CERT_KEY" EnvCertAuth = "FN_NODE_CERT_AUTHORITY" + EnvProcessCollectorList = "FN_PROCESS_COLLECTOR_LIST" + // Defaults DefaultLogLevel = "info" DefaultLogDest = "stderr" @@ -530,10 +533,17 @@ func WithPrometheus() ServerOption { return func(ctx context.Context, s *Server) error { reg := promclient.NewRegistry() reg.MustRegister(promclient.NewProcessCollector(os.Getpid(), "fn"), - promclient.NewProcessCollectorPIDFn(dockerPid(), "dockerd"), promclient.NewGoCollector(), ) + for _, exeName := range getMonitoredCmdNames() { + san := promSanitizeMetricName(exeName) + err := reg.Register(promclient.NewProcessCollectorPIDFn(getPidCmd(exeName), san)) + if err != nil { + panic(err) + } + } + exporter, err := prometheus.NewExporter(prometheus.Options{ Namespace: "fn", Registry: reg, @@ -590,8 +600,35 @@ func WithZipkin(zipkinURL string) ServerOption { } } +// prometheus only allows [a-zA-Z0-9:_] in metrics names. +func promSanitizeMetricName(name string) string { + res := make([]rune, 0, len(name)) + for _, rVal := range name { + if unicode.IsDigit(rVal) || unicode.IsLetter(rVal) || rVal == ':' { + res = append(res, rVal) + } else { + res = append(res, '_') + } + } + return string(res) +} + +// determine sidecar-monitored cmd names. But by default +// we track dockerd + containerd +func getMonitoredCmdNames() []string { + + // override? empty variable to disable trackers + val, ok := os.LookupEnv(EnvProcessCollectorList) + if ok { + return strings.Fields(val) + } + + // by default, we monitor dockerd and containerd + return []string{"dockerd", "docker-containerd"} +} + // TODO plumbing considerations, we've put the S pipe next to the chandalier... -func dockerPid() func() (int, error) { +func getPidCmd(cmdName string) func() (int, error) { // prometheus' process collector only works on linux anyway. let them do the // process detection, if we return an error here we just get 0 metrics and it // does not log / blow up (that's fine!) it's also likely we hit permissions @@ -602,50 +639,62 @@ func dockerPid() func() (int, error) { return func() (int, error) { if pid != 0 { - // make sure it's docker pid. - if isDockerPid("/proc/" + strconv.Itoa(pid) + "/status") { + // make sure it's our pid. + if isPidMatchCmd(cmdName, pid) { return pid, nil } pid = 0 // reset to go search } - err := filepath.Walk("/proc", func(path string, info os.FileInfo, err error) error { - if err != nil || pid != 0 { - // we get permission errors digging around in here, ignore them and press on - return nil - } - - // /proc//status - if strings.Count(path, "/") == 3 && strings.Contains(path, "/status") { - if isDockerPid(path) { - // extract pid from path - pid, _ = strconv.Atoi(path[6:strings.LastIndex(path, "/")]) - return io.EOF // end the search + if pids, err := getPidList(); err == nil { + for _, test := range pids { + if isPidMatchCmd(cmdName, test) { + pid = test + return pid, nil } } - - // keep searching - return nil - }) - if err == io.EOF { // used as sentinel - err = nil } - return pid, err + + return pid, io.EOF } } -func isDockerPid(path string) bool { - // first line of status file is: "Name: " - f, err := os.Open(path) +func isPidMatchCmd(cmdName string, pid int) bool { + fs, err := os.Open("/proc/" + strconv.Itoa(pid) + "/cmdline") if err != nil { return false } - defer f.Close() + defer fs.Close() - // scan first line only - scanner := bufio.NewScanner(f) - scanner.Scan() - return strings.HasSuffix(scanner.Text(), "dockerd") + rd := bufio.NewReader(fs) + tok, err := rd.ReadSlice(0) + if err != nil || len(tok) < len(cmdName) { + return false + } + + return filepath.Base(string(tok[:len(tok)-1])) == cmdName +} + +func getPidList() ([]int, error) { + var pids []int + dir, err := os.Open("/proc") + if err != nil { + return pids, nil + } + defer dir.Close() + + files, err := dir.Readdirnames(0) + if err != nil { + return pids, nil + } + + pids = make([]int, 0, len(files)) + for _, tok := range files { + if conv, err := strconv.ParseUint(tok, 10, 64); err == nil { + pids = append(pids, int(conv)) + } + } + return pids, nil } func setMachineID() {