Scrape CPU latency stats from /proc/schedstat (#1389)

These are useful as a direct indication of CPU contention and task
scheduler latency.

Handy references:
 - https://github.com/torvalds/linux/blob/master/Documentation/scheduler/sched-stats.txt
 - https://doc.opensuse.org/documentation/leap/tuning/html/book.sle.tuning/cha.tuning.taskscheduler.html

procfs is updated to pull in the enabling change:
https://github.com/prometheus/procfs/pull/186

Signed-off-by: Phil Frost <phil@postmates.com>

1 files changed, 94 insertions, 0 deletions

diff --git a/collector/schedstat_linux.go b/collector/schedstat_linux.go
new file mode 100644
index 0000000..5e7638a
--- /dev/null
+++ b/collector/schedstat_linux.go
@@ -0,0 +1,94 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+        "fmt"
+
+        "github.com/prometheus/client_golang/prometheus"
+        "github.com/prometheus/procfs"
+)
+
+var (
+        runningSecondsTotal = prometheus.NewDesc(
+                prometheus.BuildFQName(namespace, "schedstat", "running_seconds_total"),
+                "Number of seconds CPU spent running a process.",
+                []string{"cpu"},
+                nil,
+        )
+
+        waitingSecondsTotal = prometheus.NewDesc(
+                prometheus.BuildFQName(namespace, "schedstat", "waiting_seconds_total"),
+                "Number of seconds spent by processing waiting for this CPU.",
+                []string{"cpu"},
+                nil,
+        )
+
+        timeslicesTotal = prometheus.NewDesc(
+                prometheus.BuildFQName(namespace, "schedstat", "timeslices_total"),
+                "Number of timeslices executed by CPU.",
+                []string{"cpu"},
+                nil,
+        )
+)
+
+// NewSchedstatCollector returns a new Collector exposing task scheduler statistics
+func NewSchedstatCollector() (Collector, error) {
+        fs, err := procfs.NewFS(*procPath)
+        if err != nil {
+                return nil, fmt.Errorf("failed to open procfs: %v", err)
+        }
+
+        return &schedstatCollector{fs: fs}, nil
+}
+
+type schedstatCollector struct {
+        fs procfs.FS
+}
+
+func init() {
+        registerCollector("schedstat", defaultEnabled, NewSchedstatCollector)
+}
+
+func (c *schedstatCollector) Update(ch chan<- prometheus.Metric) error {
+        stats, err := c.fs.Schedstat()
+        if err != nil {
+                return err
+        }
+
+        for _, cpu := range stats.CPUs {
+                ch <- prometheus.MustNewConstMetric(
+                        runningSecondsTotal,
+                        prometheus.CounterValue,
+                        cpu.RunningSeconds(),
+                        cpu.CPUNum,
+                )
+
+                ch <- prometheus.MustNewConstMetric(
+                        waitingSecondsTotal,
+                        prometheus.CounterValue,
+                        cpu.WaitingSeconds(),
+                        cpu.CPUNum,
+                )
+
+                ch <- prometheus.MustNewConstMetric(
+                        timeslicesTotal,
+                        prometheus.CounterValue,
+                        float64(cpu.RunTimeslices),
+                        cpu.CPUNum,
+                )
+        }
+
+        return nil
+}