read contents of objset file (#1632)

* added objread functionality

Signed-off-by: Sudharshann D <sudhar287@gmail.com>

9 files changed, 238 insertions, 6 deletions

diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt
index 0ea506b..3e9f6cb 100644
--- a/collector/fixtures/e2e-64k-page-output.txt
+++ b/collector/fixtures/e2e-64k-page-output.txt
@@ -3584,6 +3584,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19
 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count
 # TYPE node_zfs_zil_zil_itx_needcopy_count untyped
 node_zfs_zil_zil_itx_needcopy_count 0
+# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread
+# TYPE node_zfs_zpool_dataset_nread untyped
+node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28
+node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28
+# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked
+# TYPE node_zfs_zpool_dataset_nunlinked untyped
+node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks
+# TYPE node_zfs_zpool_dataset_nunlinks untyped
+node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten
+# TYPE node_zfs_zpool_dataset_nwritten untyped
+node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302
+node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806
+# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads
+# TYPE node_zfs_zpool_dataset_reads untyped
+node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2
+node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2
+# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes
+# TYPE node_zfs_zpool_dataset_writes untyped
+node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4
+node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10
 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread
 # TYPE node_zfs_zpool_nread untyped
 node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06
diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt
index 5a2ba25..df04679 100644
--- a/collector/fixtures/e2e-output.txt
+++ b/collector/fixtures/e2e-output.txt
@@ -3677,6 +3677,42 @@ node_zfs_zil_zil_itx_needcopy_bytes 1.8446744073709537e+19
 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count
 # TYPE node_zfs_zil_zil_itx_needcopy_count untyped
 node_zfs_zil_zil_itx_needcopy_count 0
+# HELP node_zfs_zpool_dataset_nread kstat.zfs.misc.objset.nread
+# TYPE node_zfs_zpool_dataset_nread untyped
+node_zfs_zpool_dataset_nread{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nread{dataset="pool1/dataset1",zpool="pool1"} 28
+node_zfs_zpool_dataset_nread{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nread{dataset="poolz1/dataset1",zpool="poolz1"} 28
+# HELP node_zfs_zpool_dataset_nunlinked kstat.zfs.misc.objset.nunlinked
+# TYPE node_zfs_zpool_dataset_nunlinked untyped
+node_zfs_zpool_dataset_nunlinked{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinked{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nunlinks kstat.zfs.misc.objset.nunlinks
+# TYPE node_zfs_zpool_dataset_nunlinks untyped
+node_zfs_zpool_dataset_nunlinks{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="pool1/dataset1",zpool="pool1"} 3
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nunlinks{dataset="poolz1/dataset1",zpool="poolz1"} 14
+# HELP node_zfs_zpool_dataset_nwritten kstat.zfs.misc.objset.nwritten
+# TYPE node_zfs_zpool_dataset_nwritten untyped
+node_zfs_zpool_dataset_nwritten{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="pool1/dataset1",zpool="pool1"} 12302
+node_zfs_zpool_dataset_nwritten{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_nwritten{dataset="poolz1/dataset1",zpool="poolz1"} 32806
+# HELP node_zfs_zpool_dataset_reads kstat.zfs.misc.objset.reads
+# TYPE node_zfs_zpool_dataset_reads untyped
+node_zfs_zpool_dataset_reads{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_reads{dataset="pool1/dataset1",zpool="pool1"} 2
+node_zfs_zpool_dataset_reads{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_reads{dataset="poolz1/dataset1",zpool="poolz1"} 2
+# HELP node_zfs_zpool_dataset_writes kstat.zfs.misc.objset.writes
+# TYPE node_zfs_zpool_dataset_writes untyped
+node_zfs_zpool_dataset_writes{dataset="pool1",zpool="pool1"} 0
+node_zfs_zpool_dataset_writes{dataset="pool1/dataset1",zpool="pool1"} 4
+node_zfs_zpool_dataset_writes{dataset="poolz1",zpool="poolz1"} 0
+node_zfs_zpool_dataset_writes{dataset="poolz1/dataset1",zpool="poolz1"} 10
 # HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread
 # TYPE node_zfs_zpool_nread untyped
 node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06
diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1 b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1
new file mode 100644
index 0000000..28a9d2e
--- /dev/null
+++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-1
@@ -0,0 +1,9 @@
+23 1 0x01 7 2160 221578688875 6665999035587
+name                            type data
+dataset_name                    7    pool1
+writes                          4    0
+nwritten                        4    0
+reads                           4    0
+nread                           4    0
+nunlinks                        4    0
+nunlinked                       4    0
diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2 b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2
new file mode 100644
index 0000000..79ee31d
--- /dev/null
+++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/objset-2
@@ -0,0 +1,9 @@
+24 1 0x01 7 2160 221611904716 7145015038451
+name                            type data
+dataset_name                    7    pool1/dataset1
+writes                          4    4
+nwritten                        4    12302
+reads                           4    2
+nread                           4    28
+nunlinks                        4    3
+nunlinked                       4    3
diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1 b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1
new file mode 100644
index 0000000..189b655
--- /dev/null
+++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-1
@@ -0,0 +1,9 @@
+30 1 0x01 7 2160 217993779684 2621674546179
+name                            type data
+dataset_name                    7    poolz1
+writes                          4    0
+nwritten                        4    0
+reads                           4    0
+nread                           4    0
+nunlinks                        4    0
+nunlinked                       4    0
\ No newline at end of file
diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2 b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2
new file mode 100644
index 0000000..3d44623
--- /dev/null
+++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/objset-2
@@ -0,0 +1,9 @@
+31 1 0x01 7 2160 218133979890 3024169078920
+name                            type data
+dataset_name                    7    poolz1/dataset1
+writes                          4    10
+nwritten                        4    32806
+reads                           4    2
+nread                           4    28
+nunlinks                        4    14
+nunlinked                       4    14
\ No newline at end of file
diff --git a/collector/zfs.go b/collector/zfs.go
index 093ded5..b530e35 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -34,17 +34,19 @@ func init() {
 }
 
 type zfsCollector struct {
-        linuxProcpathBase string
-        linuxZpoolIoPath  string
-        linuxPathMap      map[string]string
-        logger            log.Logger
+        linuxProcpathBase    string
+        linuxZpoolIoPath     string
+        linuxZpoolObjsetPath string
+        linuxPathMap         map[string]string
+        logger               log.Logger
 }
 
 // NewZFSCollector returns a new Collector exposing ZFS statistics.
 func NewZFSCollector(logger log.Logger) (Collector, error) {
         return &zfsCollector{
-                linuxProcpathBase: "spl/kstat/zfs",
-                linuxZpoolIoPath:  "/*/io",
+                linuxProcpathBase:    "spl/kstat/zfs",
+                linuxZpoolIoPath:     "/*/io",
+                linuxZpoolObjsetPath: "/*/objset-*",
                 linuxPathMap: map[string]string{
                         "zfs_abd":         "abdstats",
                         "zfs_arc":         "arcstats",
@@ -113,3 +115,20 @@ func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value
                 poolName,
         )
 }
+
+func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string, sysctl zfsSysctl, value uint64) prometheus.Metric {
+        metricName := sysctl.metricName()
+
+        return prometheus.MustNewConstMetric(
+                prometheus.NewDesc(
+                        prometheus.BuildFQName(namespace, "zfs_zpool_dataset", metricName),
+                        string(sysctl),
+                        []string{"zpool", "dataset"},
+                        nil,
+                ),
+                prometheus.UntypedValue,
+                float64(value),
+                poolName,
+                datasetName,
+        )
+}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index aa27db8..e2c9749 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -90,6 +90,31 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
                 }
         }
 
+        zpoolObjsetPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolObjsetPath)))
+        if err != nil {
+                return err
+        }
+
+        if zpoolObjsetPaths == nil {
+                return nil
+        }
+
+        for _, zpoolPath := range zpoolObjsetPaths {
+                file, err := os.Open(zpoolPath)
+                if err != nil {
+                        // this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore
+                        level.Debug(c.logger).Log("msg", "Cannot open file for reading", "path", zpoolPath)
+                        return errZFSNotAvailable
+                }
+
+                err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+                        ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v)
+                })
+                file.Close()
+                if err != nil {
+                        return err
+                }
+        }
         return nil
 }
 
@@ -168,3 +193,43 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h
 
         return scanner.Err()
 }
+
+func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error {
+        scanner := bufio.NewScanner(reader)
+
+        parseLine := false
+        var zpoolName, datasetName string
+        for scanner.Scan() {
+                parts := strings.Fields(scanner.Text())
+
+                if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
+                        parseLine = true
+                        continue
+                }
+
+                if !parseLine || len(parts) < 3 {
+                        continue
+                }
+                if parts[0] == "dataset_name" {
+                        zpoolPathElements := strings.Split(zpoolPath, "/")
+                        pathLen := len(zpoolPathElements)
+                        zpoolName = zpoolPathElements[pathLen-2]
+                        datasetName = parts[2]
+                        continue
+                }
+
+                if parts[1] == kstatDataUint64 {
+                        key := fmt.Sprintf("kstat.zfs.misc.objset.%s", parts[0])
+                        value, err := strconv.ParseUint(parts[2], 10, 64)
+                        if err != nil {
+                                return fmt.Errorf("could not parse expected integer value for %q", key)
+                        }
+                        handler(zpoolName, datasetName, zfsSysctl(key), value)
+                }
+        }
+        if !parseLine {
+                return fmt.Errorf("did not parse a single %s %s metric", zpoolName, datasetName)
+        }
+
+        return scanner.Err()
+}
diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go
index 3574858..29d28e8 100644
--- a/collector/zfs_linux_test.go
+++ b/collector/zfs_linux_test.go
@@ -311,6 +311,46 @@ func TestZpoolParsing(t *testing.T) {
         }
 }
 
+func TestZpoolObjsetParsing(t *testing.T) {
+        zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/objset-*")
+        if err != nil {
+                t.Fatal(err)
+        }
+
+        c := zfsCollector{}
+        if err != nil {
+                t.Fatal(err)
+        }
+
+        handlerCalled := false
+        for _, zpoolPath := range zpoolPaths {
+                file, err := os.Open(zpoolPath)
+                if err != nil {
+                        t.Fatal(err)
+                }
+
+                err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) {
+                        if s != zfsSysctl("kstat.zfs.misc.objset.writes") {
+                                return
+                        }
+
+                        handlerCalled = true
+
+                        if v != uint64(0) && v != uint64(4) && v != uint64(10) {
+                                t.Fatalf("Incorrect value parsed from procfs data %v", v)
+                        }
+
+                })
+                file.Close()
+                if err != nil {
+                        t.Fatal(err)
+                }
+        }
+        if !handlerCalled {
+                t.Fatal("Zpool parsing handler was not called for some expected sysctls")
+        }
+}
+
 func TestAbdstatsParsing(t *testing.T) {
         abdstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/abdstats")
         if err != nil {