fn: add docker-containerd to prometheus processes (#863)

*) switch to /proc/<pid>/cmdline due to /proc/<pid>/status
process name truncation.
*) Optional override with FN_PROCESS_COLLECTOR_LIST

api/server/server.go

@@ -17,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"unicode"
 
 	"github.com/fnproject/fn/api/agent"
 	"github.com/fnproject/fn/api/agent/hybrid"
@@ -63,6 +64,8 @@ const (
 	EnvCertKey  = "FN_NODE_CERT_KEY"
 	EnvCertAuth = "FN_NODE_CERT_AUTHORITY"
 
+	EnvProcessCollectorList = "FN_PROCESS_COLLECTOR_LIST"
+
 	// Defaults
 	DefaultLogLevel = "info"
 	DefaultLogDest  = "stderr"
@@ -530,10 +533,17 @@ func WithPrometheus() ServerOption {
 	return func(ctx context.Context, s *Server) error {
 		reg := promclient.NewRegistry()
 		reg.MustRegister(promclient.NewProcessCollector(os.Getpid(), "fn"),
-			promclient.NewProcessCollectorPIDFn(dockerPid(), "dockerd"),
 			promclient.NewGoCollector(),
 		)
 
+		for _, exeName := range getMonitoredCmdNames() {
+			san := promSanitizeMetricName(exeName)
+			err := reg.Register(promclient.NewProcessCollectorPIDFn(getPidCmd(exeName), san))
+			if err != nil {
+				panic(err)
+			}
+		}
+
 		exporter, err := prometheus.NewExporter(prometheus.Options{
 			Namespace: "fn",
 			Registry:  reg,
@@ -590,8 +600,35 @@ func WithZipkin(zipkinURL string) ServerOption {
 	}
 }
 
+// prometheus only allows [a-zA-Z0-9:_] in metrics names.
+func promSanitizeMetricName(name string) string {
+	res := make([]rune, 0, len(name))
+	for _, rVal := range name {
+		if unicode.IsDigit(rVal) || unicode.IsLetter(rVal) || rVal == ':' {
+			res = append(res, rVal)
+		} else {
+			res = append(res, '_')
+		}
+	}
+	return string(res)
+}
+
+// determine sidecar-monitored cmd names. But by default
+// we track dockerd + containerd
+func getMonitoredCmdNames() []string {
+
+	// override? empty variable to disable trackers
+	val, ok := os.LookupEnv(EnvProcessCollectorList)
+	if ok {
+		return strings.Fields(val)
+	}
+
+	// by default, we monitor dockerd and containerd
+	return []string{"dockerd", "docker-containerd"}
+}
+
 // TODO plumbing considerations, we've put the S pipe next to the chandalier...
-func dockerPid() func() (int, error) {
+func getPidCmd(cmdName string) func() (int, error) {
 	// prometheus' process collector only works on linux anyway. let them do the
 	// process detection, if we return an error here we just get 0 metrics and it
 	// does not log / blow up (that's fine!) it's also likely we hit permissions
@@ -602,50 +639,62 @@ func dockerPid() func() (int, error) {
 
 	return func() (int, error) {
 		if pid != 0 {
-			// make sure it's docker pid.
-			if isDockerPid("/proc/" + strconv.Itoa(pid) + "/status") {
+			// make sure it's our pid.
+			if isPidMatchCmd(cmdName, pid) {
 				return pid, nil
 			}
 			pid = 0 // reset to go search
 		}
 
-		err := filepath.Walk("/proc", func(path string, info os.FileInfo, err error) error {
-			if err != nil || pid != 0 {
-				// we get permission errors digging around in here, ignore them and press on
-				return nil
-			}
-
-			// /proc/<pid>/status
-			if strings.Count(path, "/") == 3 && strings.Contains(path, "/status") {
-				if isDockerPid(path) {
-					// extract pid from path
-					pid, _ = strconv.Atoi(path[6:strings.LastIndex(path, "/")])
-					return io.EOF // end the search
+		if pids, err := getPidList(); err == nil {
+			for _, test := range pids {
+				if isPidMatchCmd(cmdName, test) {
+					pid = test
+					return pid, nil
 				}
 			}
-
-			// keep searching
-			return nil
-		})
-		if err == io.EOF { // used as sentinel
-			err = nil
 		}
-		return pid, err
+
+		return pid, io.EOF
 	}
 }
 
-func isDockerPid(path string) bool {
-	// first line of status file is: "Name: <name>"
-	f, err := os.Open(path)
+func isPidMatchCmd(cmdName string, pid int) bool {
+	fs, err := os.Open("/proc/" + strconv.Itoa(pid) + "/cmdline")
 	if err != nil {
 		return false
 	}
-	defer f.Close()
+	defer fs.Close()
 
-	// scan first line only
-	scanner := bufio.NewScanner(f)
-	scanner.Scan()
-	return strings.HasSuffix(scanner.Text(), "dockerd")
+	rd := bufio.NewReader(fs)
+	tok, err := rd.ReadSlice(0)
+	if err != nil || len(tok) < len(cmdName) {
+		return false
+	}
+
+	return filepath.Base(string(tok[:len(tok)-1])) == cmdName
+}
+
+func getPidList() ([]int, error) {
+	var pids []int
+	dir, err := os.Open("/proc")
+	if err != nil {
+		return pids, nil
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdirnames(0)
+	if err != nil {
+		return pids, nil
+	}
+
+	pids = make([]int, 0, len(files))
+	for _, tok := range files {
+		if conv, err := strconv.ParseUint(tok, 10, 64); err == nil {
+			pids = append(pids, int(conv))
+		}
+	}
+	return pids, nil
 }
 
 func setMachineID() {