zfs: introduce pkg zfs/zfscmd for command logging, status, prometheus metrics
refs #196
This commit is contained in:
Christian Schwarz
@@ -0,0 +1,11 @@
|
||||
The tool in this package (`go run . -h`) scrapes log lines produces by the `github.com/zrepl/zrepl/zfs/zfscmd` package
|
||||
into a stream of JSON objects.
|
||||
|
||||
The `analysis.ipynb` then runs some basic analysis on the collected log output.
|
||||
|
||||
## Deps for the `scrape_graylog_csv.bash` script
|
||||
|
||||
```
|
||||
pip install --upgrade git+https://github.com/lk-jeffpeck/csvfilter.git@ec433f14330fbbf5d41f56febfeedac22868a949
|
||||
```
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import seaborn as sns\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import re\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ./parsed.json is the stdout of the scraper tool in this directory\n",
|
||||
"df = pd.read_json(\"./parsed.json\", lines=True)\n",
|
||||
"df"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def parse_ds(entity):\n",
|
||||
" m = re.search(r\"(?P<dataset>[^@#]*)([@#].+)?\", entity)\n",
|
||||
" return m.group(\"dataset\")\n",
|
||||
" \n",
|
||||
"def parse_cmd(row):\n",
|
||||
" cmd = row.Cmd\n",
|
||||
" binary, verb, *tail = re.split(r\"\\s+\", cmd) # NOTE whitespace in dataset names => don't use comp\n",
|
||||
" \n",
|
||||
" dataset = None\n",
|
||||
" if binary == \"zfs\":\n",
|
||||
" if verb == \"send\": \n",
|
||||
" if len(tail) == 0:\n",
|
||||
" verb = \"send-feature-test\"\n",
|
||||
" else:\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" if \"-n\" in tail:\n",
|
||||
" verb = \"send-dry\"\n",
|
||||
" elif verb == \"recv\" or verb == \"receive\":\n",
|
||||
" verb = \"receive\"\n",
|
||||
" if len(tail) > 0:\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" else:\n",
|
||||
" verb = \"receive-CLI-test\"\n",
|
||||
" elif verb == \"get\":\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" elif verb == \"list\":\n",
|
||||
" if \"-r\" in tail and \"-d\" in tail and \"1\" in tail:\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" verb = \"list-single-dataset\"\n",
|
||||
" else:\n",
|
||||
" dataset = \"!ALL_POOLS!\"\n",
|
||||
" verb = \"list-all-filesystems\"\n",
|
||||
" elif verb == \"bookmark\":\n",
|
||||
" dataset = parse_ds(tail[-2])\n",
|
||||
" elif verb == \"hold\":\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" elif verb == \"snapshot\":\n",
|
||||
" dataset = parse_ds(tail[-1])\n",
|
||||
" elif verb == \"release\":\n",
|
||||
" dss = tail[-1].split(\",\")\n",
|
||||
" if len(dss) > 1:\n",
|
||||
" raise Exception(\"cannot handle batch-release\")\n",
|
||||
" dataset = parse_ds(dss[0])\n",
|
||||
" elif verb == \"holds\" and \"-H\" in tail:\n",
|
||||
" dss = tail[-1].split(\",\")\n",
|
||||
" if len(dss) > 1:\n",
|
||||
" raise Exception(\"cannot handle batch-holds\")\n",
|
||||
" dataset = parse_ds(dss[0])\n",
|
||||
" elif verb == \"destroy\":\n",
|
||||
" dss = tail[-1].split(\",\")\n",
|
||||
" if len(dss) > 1:\n",
|
||||
" raise Exception(\"cannot handle batch-holds\")\n",
|
||||
" dataset = parse_ds(dss[0])\n",
|
||||
" \n",
|
||||
" return {'action':binary + \"-\" + verb, 'dataset': dataset }\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"res = df.apply(parse_cmd, axis='columns', result_type='expand')\n",
|
||||
"res = pd.concat([df, res], axis='columns')\n",
|
||||
"for cat in [\"action\", \"dataset\"]:\n",
|
||||
" res[cat] = res[cat].astype('category')\n",
|
||||
"\n",
|
||||
"res[\"LogTimeUnix\"] = pd.to_datetime(res.LogTime)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"res[\"OtherTime\"] = res.TotalTime - res.Usertime - res.Systime\n",
|
||||
"x = res.melt(id_vars=[\"action\", \"dataset\"], value_vars=[\"TotalTime\", \"OtherTime\", \"Usertime\", \"Systime\"])\n",
|
||||
"x"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"commands with NaN values\")\n",
|
||||
"set(x[x.isna().any(axis=1)].action.values)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# (~x.action.astype('str').isin([\"zfs-send\", \"zfs-recv\"]))\n",
|
||||
"totaltimes = x[(x.variable == \"TotalTime\")].groupby([\"action\", \"dataset\"]).sum().reset_index()\n",
|
||||
"display(totaltimes)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"totaltimes_by_action = totaltimes.groupby(\"action\").sum().sort_values(by=\"value\")\n",
|
||||
"totaltimes_by_action.plot.barh()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"totaltimes.groupby(\"dataset\").sum().plot.barh(fontsize=5)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"most_expensive_action = totaltimes_by_action.idxmax().value\n",
|
||||
"display(most_expensive_action)\n",
|
||||
"most_expensive_action_by_dataset = totaltimes[totaltimes.action == most_expensive_action].groupby(\"dataset\").sum().sort_values(by=\"value\")\n",
|
||||
"most_expensive_action_by_dataset.plot.barh(rot=50, fontsize=5, figsize=(10, 20))\n",
|
||||
"plt.savefig('most-expensive-command.pdf')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"res"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %matplotlib notebook \n",
|
||||
"\n",
|
||||
"# res.index = res.LogTimeUnix\n",
|
||||
"\n",
|
||||
"# resampled = res.pivot(columns='action', values='TotalTime').resample(\"1s\").sum()\n",
|
||||
"# resampled.cumsum().plot()\n",
|
||||
"# res[\"BeginTime\"] = res.LogTimeUnix.dt.total_seconds()\n",
|
||||
"# holds = res[res.action == \"zfs-holds\"]\n",
|
||||
"# sns.stripplot(x=\"LogTimeUnix\", y=\"action\", data=res)\n",
|
||||
"# res[\"LogTimeUnix\"].resample(\"20min\").sum()\n",
|
||||
"# res[res.action == \"zfs-holds\"].plot.scatter(x=\"LogTimeUnix\", y=\"TotalTime\")\n",
|
||||
"\n",
|
||||
"#res[res.action == \"zfs-holds\"].pivot(columns='action', values=['TotalTime', 'Systime', \"Usertime\"]).resample(\"1s\").sum().cumsum().plot()\n",
|
||||
"pivoted = res.reset_index(drop=True).pivot_table(values=['TotalTime', 'Systime', \"Usertime\"], index=\"LogTimeUnix\", columns=\"action\")\n",
|
||||
"pivoted"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pivoted.cumsum()[[(\"TotalTime\", \"zfs-holds\"),(\"Systime\", \"zfs-holds\"),(\"Usertime\", \"zfs-holds\")]].plot()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pivoted = res.reset_index(drop=True).pivot_table(values=['TotalTime'], index=\"LogTimeUnix\", columns=\"action\")\n",
|
||||
"cum_invocation_counts_per_action = pivoted.isna().astype(int).cumsum()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"cum_invocation_counts_per_action"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# zfs-get as reference value\n",
|
||||
"cum_invocation_counts_per_action[[(\"TotalTime\",\"zfs-holds\"),(\"TotalTime\",\"zfs-get\")]].plot()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This script converts output that was produced by zrepl and captured by Graylog
|
||||
# back to something that the scraper in this package's main can understand
|
||||
# Intended for human syslog
|
||||
# logging:
|
||||
# - type: syslog
|
||||
# level: debug
|
||||
# format: human
|
||||
|
||||
|
||||
csvfilter --skip 1 -f 0,2 -q '"' --out-quotechar=' ' /dev/stdin | sed -E 's/^\s*([^,]*), /\1 [LEVEL]/' | \
|
||||
go run . -v \
|
||||
--dateRE '^([^\[]+) (\[.*)' \
|
||||
--dateFormat '2006-01-02T15:04:05.999999999Z'
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-logfmt/logfmt"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
)
|
||||
|
||||
type RuntimeLine struct {
|
||||
LogTime time.Time
|
||||
Cmd string
|
||||
TotalTime, Usertime, Systime time.Duration
|
||||
Error string
|
||||
}
|
||||
|
||||
var humanFormatterLineRE = regexp.MustCompile(`^(\[[^\]]+\]){2}\[zfs.cmd\]:\s+command\s+exited\s+(with|without)\s+error\s+(.+)`)
|
||||
|
||||
func parseSecs(s string) (time.Duration, error) {
|
||||
d, err := time.ParseDuration(s + "s")
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "parse duration %q", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func parseHumanFormatterNodate(line string) (l RuntimeLine, err error) {
|
||||
m := humanFormatterLineRE.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return l, errors.New("human formatter regex does not match")
|
||||
}
|
||||
|
||||
d := logfmt.NewDecoder(strings.NewReader(m[3]))
|
||||
for d.ScanRecord() {
|
||||
for d.ScanKeyval() {
|
||||
k := string(d.Key())
|
||||
v := string(d.Value())
|
||||
switch k {
|
||||
case "cmd":
|
||||
l.Cmd = v
|
||||
case "total_time_s":
|
||||
l.TotalTime, err = parseSecs(v)
|
||||
case "usertime_s":
|
||||
l.Usertime, err = parseSecs(v)
|
||||
case "systemtime_s":
|
||||
l.Systime, err = parseSecs(v)
|
||||
case "err":
|
||||
l.Error = v
|
||||
case "invocation":
|
||||
continue // pass
|
||||
default:
|
||||
return l, errors.Errorf("unknown key %q", k)
|
||||
}
|
||||
if err != nil {
|
||||
return l, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if d.Err() != nil {
|
||||
return l, errors.Wrap(d.Err(), "decode key value pairs")
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func parseLogLine(line string) (l RuntimeLine, err error) {
|
||||
m := dateRegex.FindStringSubmatch(line)
|
||||
if len(m) != 3 {
|
||||
return l, errors.Errorf("invalid date regex match %v", m)
|
||||
}
|
||||
dateTrimmed := strings.TrimSpace(m[1])
|
||||
date, err := time.Parse(dateFormat, dateTrimmed)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("cannot parse date %q: %s", dateTrimmed, err))
|
||||
}
|
||||
logLine := m[2]
|
||||
|
||||
l, err = parseHumanFormatterNodate(strings.TrimSpace(logLine))
|
||||
l.LogTime = date
|
||||
return l, err
|
||||
}
|
||||
|
||||
var verbose bool
|
||||
var dateRegexArg string
|
||||
var dateRegex = regexp.MustCompile(`^([^\[]+)(.*)`)
|
||||
var dateFormat = logging.HumanFormatterDateFormat
|
||||
|
||||
func main() {
|
||||
|
||||
pflag.StringVarP(&dateRegexArg, "dateRE", "d", "", "date regex")
|
||||
pflag.StringVar(&dateFormat, "dateFormat", logging.HumanFormatterDateFormat, "go date format")
|
||||
pflag.BoolVarP(&verbose, "verbose", "v", false, "verbose")
|
||||
pflag.Parse()
|
||||
|
||||
if dateRegexArg != "" {
|
||||
dateRegex = regexp.MustCompile(dateRegexArg)
|
||||
}
|
||||
|
||||
input := bufio.NewScanner(os.Stdin)
|
||||
input.Split(bufio.ScanLines)
|
||||
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
for input.Scan() {
|
||||
|
||||
l, err := parseLogLine(input.Text())
|
||||
if err != nil && verbose {
|
||||
fmt.Fprintf(os.Stderr, "ignoring line after error %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "offending line was: %s\n", input.Text())
|
||||
}
|
||||
if err == nil {
|
||||
if err := enc.Encode(l); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if input.Err() != nil {
|
||||
panic(input.Err())
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
)
|
||||
|
||||
func TestParseHumanFormatter(t *testing.T) {
|
||||
|
||||
type testCase struct {
|
||||
Name string
|
||||
Input string
|
||||
Expect *RuntimeLine
|
||||
ExpectErr string
|
||||
}
|
||||
|
||||
secs := func(s string) time.Duration {
|
||||
d, err := parseSecs(s)
|
||||
require.NoError(t, err)
|
||||
return d
|
||||
}
|
||||
|
||||
logTime, err := time.Parse(logging.HumanFormatterDateFormat, "2020-04-04T00:00:05+02:00")
|
||||
require.NoError(t, err)
|
||||
|
||||
tcs := []testCase{
|
||||
{
|
||||
Name: "human-formatter-noerror",
|
||||
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd]: command exited without error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619"`,
|
||||
Expect: &RuntimeLine{
|
||||
Cmd: "zfs list -H -p -o name -r -t filesystem,volume",
|
||||
TotalTime: secs("0.037828619"),
|
||||
Usertime: secs("0.008445"),
|
||||
Systime: secs("0.033783"),
|
||||
Error: "",
|
||||
LogTime: logTime,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "human-formatter-witherror",
|
||||
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd]: command exited with error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619" err="some error"`,
|
||||
Expect: &RuntimeLine{
|
||||
Cmd: "zfs list -H -p -o name -r -t filesystem,volume",
|
||||
TotalTime: secs("0.037828619"),
|
||||
Usertime: secs("0.008445"),
|
||||
Systime: secs("0.033783"),
|
||||
Error: "some error",
|
||||
LogTime: logTime,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "from graylog",
|
||||
Input: `2020-04-04T00:00:05+02:00 [DEBG][csnas][zfs.cmd]: command exited without error usertime_s="0" cmd="zfs send -i zroot/ezjail/synapse-12@zrepl_20200329_095518_000 zroot/ezjail/synapse-12@zrepl_20200329_102454_000" total_time_s="0.101598591" invocation="85" systemtime_s="0.041581"`,
|
||||
Expect: &RuntimeLine{
|
||||
Cmd: "zfs send -i zroot/ezjail/synapse-12@zrepl_20200329_095518_000 zroot/ezjail/synapse-12@zrepl_20200329_102454_000",
|
||||
TotalTime: secs("0.101598591"),
|
||||
Systime: secs("0.041581"),
|
||||
Usertime: secs("0"),
|
||||
Error: "",
|
||||
LogTime: logTime,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range tcs {
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
l, err := parseLogLine(c.Input)
|
||||
t.Logf("l=%v", l)
|
||||
t.Logf("err=%T %v", err, err)
|
||||
if (c.Expect != nil && c.ExpectErr != "") || (c.Expect == nil && c.ExpectErr == "") {
|
||||
t.Fatal("bad test case", c)
|
||||
}
|
||||
if c.Expect != nil {
|
||||
require.Equal(t, *c.Expect, l)
|
||||
}
|
||||
if c.ExpectErr != "" {
|
||||
require.EqualError(t, err, c.ExpectErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Package zfscmd provides a wrapper around packate os/exec.
|
||||
// Functionality provided by the wrapper:
|
||||
// - logging start and end of command execution
|
||||
// - status report of active commands
|
||||
// - prometheus metrics of runtimes
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/util/circlog"
|
||||
)
|
||||
|
||||
type Cmd struct {
|
||||
cmd *exec.Cmd
|
||||
ctx context.Context
|
||||
mtx sync.RWMutex
|
||||
startedAt, waitReturnedAt time.Time
|
||||
}
|
||||
|
||||
func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
|
||||
cmd := exec.CommandContext(ctx, name, arg...)
|
||||
return &Cmd{cmd: cmd, ctx: ctx}
|
||||
}
|
||||
|
||||
// err.(*exec.ExitError).Stderr will NOT be set
|
||||
func (c *Cmd) CombinedOutput() (o []byte, err error) {
|
||||
c.startPre()
|
||||
c.startPost(nil)
|
||||
c.waitPre()
|
||||
o, err = c.cmd.CombinedOutput()
|
||||
c.waitPost(err)
|
||||
return
|
||||
}
|
||||
|
||||
// err.(*exec.ExitError).Stderr will be set
|
||||
func (c *Cmd) Output() (o []byte, err error) {
|
||||
c.startPre()
|
||||
c.startPost(nil)
|
||||
c.waitPre()
|
||||
o, err = c.cmd.Output()
|
||||
c.waitPost(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Careful: err.(*exec.ExitError).Stderr will not be set, even if you don't open an StderrPipe
|
||||
func (c *Cmd) StdoutPipeWithErrorBuf() (p io.ReadCloser, errBuf *circlog.CircularLog, err error) {
|
||||
p, err = c.cmd.StdoutPipe()
|
||||
errBuf = circlog.MustNewCircularLog(1 << 15)
|
||||
c.cmd.Stderr = errBuf
|
||||
return p, errBuf, err
|
||||
}
|
||||
|
||||
type Stdio struct {
|
||||
Stdin io.ReadCloser
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
}
|
||||
|
||||
func (c *Cmd) SetStdio(stdio Stdio) {
|
||||
c.cmd.Stdin = stdio.Stdin
|
||||
c.cmd.Stderr = stdio.Stderr
|
||||
c.cmd.Stdout = stdio.Stdout
|
||||
}
|
||||
|
||||
func (c *Cmd) String() string {
|
||||
return strings.Join(c.cmd.Args, " ") // includes argv[0] if initialized with CommandContext, that's the only way we o it
|
||||
}
|
||||
|
||||
func (c *Cmd) log() Logger {
|
||||
return getLogger(c.ctx).WithField("cmd", c.String())
|
||||
}
|
||||
|
||||
func (c *Cmd) Start() (err error) {
|
||||
c.startPre()
|
||||
err = c.cmd.Start()
|
||||
c.startPost(err)
|
||||
return err
|
||||
}
|
||||
|
||||
// only call this after a successful call to .Start()
|
||||
func (c *Cmd) Process() *os.Process {
|
||||
if c.startedAt.IsZero() {
|
||||
panic("calling Process() only allowed after successful call to Start()")
|
||||
}
|
||||
return c.cmd.Process
|
||||
}
|
||||
|
||||
func (c *Cmd) Wait() (err error) {
|
||||
c.waitPre()
|
||||
err = c.cmd.Wait()
|
||||
if !c.waitReturnedAt.IsZero() {
|
||||
// ignore duplicate waits
|
||||
return err
|
||||
}
|
||||
c.waitPost(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cmd) startPre() {
|
||||
startPreLogging(c, time.Now())
|
||||
}
|
||||
|
||||
func (c *Cmd) startPost(err error) {
|
||||
now := time.Now()
|
||||
|
||||
c.mtx.Lock()
|
||||
c.startedAt = now
|
||||
c.mtx.Unlock()
|
||||
|
||||
startPostReport(c, err, now)
|
||||
startPostLogging(c, err, now)
|
||||
}
|
||||
|
||||
func (c *Cmd) waitPre() {
|
||||
waitPreLogging(c, time.Now())
|
||||
}
|
||||
|
||||
func (c *Cmd) waitPost(err error) {
|
||||
now := time.Now()
|
||||
|
||||
c.mtx.Lock()
|
||||
c.waitReturnedAt = now
|
||||
c.mtx.Unlock()
|
||||
|
||||
waitPostReport(c, now)
|
||||
waitPostLogging(c, err, now)
|
||||
waitPostPrometheus(c, err, now)
|
||||
}
|
||||
|
||||
// returns 0 if the command did not yet finish
|
||||
func (c *Cmd) Runtime() time.Duration {
|
||||
if c.waitReturnedAt.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return c.waitReturnedAt.Sub(c.startedAt)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyLogger contextKey = iota
|
||||
contextKeyJobID
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
func WithJobID(ctx context.Context, jobID string) context.Context {
|
||||
return context.WithValue(ctx, contextKeyJobID, jobID)
|
||||
}
|
||||
|
||||
func getJobIDOrDefault(ctx context.Context, def string) string {
|
||||
ret, ok := ctx.Value(contextKeyJobID).(string)
|
||||
if !ok {
|
||||
return def
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLogger, log)
|
||||
}
|
||||
|
||||
func getLogger(ctx context.Context) Logger {
|
||||
if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
|
||||
return l
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Implementation Note:
|
||||
//
|
||||
// Pre-events logged with debug
|
||||
// Post-event without error logged with info
|
||||
// Post-events with error _also_ logged with info
|
||||
// (Not all errors we observe at this layer) are actual errors in higher-level layers)
|
||||
|
||||
func startPreLogging(c *Cmd, now time.Time) {
|
||||
c.log().Debug("starting command")
|
||||
}
|
||||
|
||||
func startPostLogging(c *Cmd, err error, now time.Time) {
|
||||
if err == nil {
|
||||
c.log().Info("started command")
|
||||
} else {
|
||||
c.log().WithError(err).Error("cannot start command")
|
||||
}
|
||||
}
|
||||
|
||||
func waitPreLogging(c *Cmd, now time.Time) {
|
||||
c.log().Debug("start waiting")
|
||||
}
|
||||
|
||||
func waitPostLogging(c *Cmd, err error, now time.Time) {
|
||||
log := c.log().
|
||||
WithField("total_time_s", c.Runtime().Seconds()).
|
||||
WithField("systemtime_s", c.cmd.ProcessState.SystemTime().Seconds()).
|
||||
WithField("usertime_s", c.cmd.ProcessState.UserTime().Seconds())
|
||||
|
||||
if err == nil {
|
||||
log.Info("command exited without error")
|
||||
} else {
|
||||
log.WithError(err).Info("command exited with error")
|
||||
}
|
||||
}
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "to stderr" 1>&2
|
||||
echo "to stdout"
|
||||
|
||||
exit "$1"
|
||||
@@ -0,0 +1,60 @@
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testBin = "./zfscmd_platform_test.bash"
|
||||
|
||||
func TestCmdStderrBehaviorOutput(t *testing.T) {
|
||||
|
||||
stdout, err := exec.Command(testBin, "0").Output()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("to stdout\n"), stdout)
|
||||
|
||||
stdout, err = exec.Command(testBin, "1").Output()
|
||||
assert.Equal(t, []byte("to stdout\n"), stdout)
|
||||
require.Error(t, err)
|
||||
ee, ok := err.(*exec.ExitError)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, ee.Stderr, []byte("to stderr\n"))
|
||||
}
|
||||
|
||||
func TestCmdStderrBehaviorCombinedOutput(t *testing.T) {
|
||||
|
||||
stdio, err := exec.Command(testBin, "0").CombinedOutput()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "to stderr\nto stdout\n", string(stdio))
|
||||
|
||||
stdio, err = exec.Command(testBin, "1").CombinedOutput()
|
||||
require.Equal(t, "to stderr\nto stdout\n", string(stdio))
|
||||
require.Error(t, err)
|
||||
ee, ok := err.(*exec.ExitError)
|
||||
require.True(t, ok)
|
||||
require.Empty(t, ee.Stderr) // !!!! maybe not what one would expect
|
||||
}
|
||||
|
||||
func TestCmdStderrBehaviorStdoutPipe(t *testing.T) {
|
||||
cmd := exec.Command(testBin, "1")
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
require.NoError(t, err)
|
||||
err = cmd.Start()
|
||||
require.NoError(t, err)
|
||||
defer cmd.Wait()
|
||||
var stdout bytes.Buffer
|
||||
_, err = io.Copy(&stdout, stdoutPipe)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "to stdout\n", stdout.String())
|
||||
|
||||
err = cmd.Wait()
|
||||
require.Error(t, err)
|
||||
ee, ok := err.(*exec.ExitError)
|
||||
require.True(t, ok)
|
||||
require.Empty(t, ee.Stderr) // !!!!! probably not what one would expect if we only redirect stdout
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var metrics struct {
|
||||
totaltime *prometheus.HistogramVec
|
||||
systemtime *prometheus.HistogramVec
|
||||
usertime *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
var timeLabels = []string{"jobid", "zfsbinary", "zfsverb"}
|
||||
var timeBuckets = []float64{0.01, 0.1, 0.2, 0.5, 0.75, 1, 2, 5, 10, 60}
|
||||
|
||||
func init() {
|
||||
metrics.totaltime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "zfscmd",
|
||||
Name: "runtime",
|
||||
Help: "number of seconds that the command took from start until wait returned",
|
||||
Buckets: timeBuckets,
|
||||
}, timeLabels)
|
||||
metrics.systemtime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "zfscmd",
|
||||
Name: "systemtime",
|
||||
Help: "https://golang.org/pkg/os/#ProcessState.SystemTime",
|
||||
Buckets: timeBuckets,
|
||||
}, timeLabels)
|
||||
metrics.usertime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "zfscmd",
|
||||
Name: "usertime",
|
||||
Help: "https://golang.org/pkg/os/#ProcessState.UserTime",
|
||||
Buckets: timeBuckets,
|
||||
}, timeLabels)
|
||||
|
||||
}
|
||||
|
||||
func RegisterMetrics(r prometheus.Registerer) {
|
||||
r.MustRegister(metrics.totaltime)
|
||||
r.MustRegister(metrics.systemtime)
|
||||
r.MustRegister(metrics.usertime)
|
||||
}
|
||||
|
||||
func waitPostPrometheus(c *Cmd, err error, now time.Time) {
|
||||
|
||||
if len(c.cmd.Args) < 2 {
|
||||
getLogger(c.ctx).WithField("args", c.cmd.Args).
|
||||
Warn("prometheus: cannot turn zfs command into metric")
|
||||
return
|
||||
}
|
||||
|
||||
// Note: do not start parsing other aspects
|
||||
// of the ZFS command line. This is not the suitable layer
|
||||
// for such a task.
|
||||
|
||||
jobid := getJobIDOrDefault(c.ctx, "_nojobid")
|
||||
|
||||
labelValues := []string{jobid, c.cmd.Args[0], c.cmd.Args[1]}
|
||||
|
||||
metrics.totaltime.
|
||||
WithLabelValues(labelValues...).
|
||||
Observe(c.Runtime().Seconds())
|
||||
metrics.systemtime.WithLabelValues(labelValues...).
|
||||
Observe(c.cmd.ProcessState.SystemTime().Seconds())
|
||||
metrics.usertime.WithLabelValues(labelValues...).
|
||||
Observe(c.cmd.ProcessState.UserTime().Seconds())
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package zfscmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Report struct {
|
||||
Active []ActiveCommand
|
||||
}
|
||||
|
||||
type ActiveCommand struct {
|
||||
Path string
|
||||
Args []string
|
||||
StartedAt time.Time
|
||||
}
|
||||
|
||||
func GetReport() *Report {
|
||||
active.mtx.RLock()
|
||||
defer active.mtx.RUnlock()
|
||||
var activeCommands []ActiveCommand
|
||||
for c := range active.cmds {
|
||||
c.mtx.RLock()
|
||||
activeCommands = append(activeCommands, ActiveCommand{
|
||||
Path: c.cmd.Path,
|
||||
Args: c.cmd.Args,
|
||||
StartedAt: c.startedAt,
|
||||
})
|
||||
c.mtx.RUnlock()
|
||||
}
|
||||
return &Report{
|
||||
Active: activeCommands,
|
||||
}
|
||||
}
|
||||
|
||||
var active struct {
|
||||
mtx sync.RWMutex
|
||||
cmds map[*Cmd]bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
active.cmds = make(map[*Cmd]bool)
|
||||
}
|
||||
|
||||
func startPostReport(c *Cmd, err error, now time.Time) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
active.mtx.Lock()
|
||||
prev := active.cmds[c]
|
||||
if prev {
|
||||
panic("impl error: duplicate active command")
|
||||
}
|
||||
active.cmds[c] = true
|
||||
active.mtx.Unlock()
|
||||
}
|
||||
|
||||
func waitPostReport(c *Cmd, now time.Time) {
|
||||
active.mtx.Lock()
|
||||
defer active.mtx.Unlock()
|
||||
prev := active.cmds[c]
|
||||
if !prev {
|
||||
panic(fmt.Sprintf("impl error: onWaitDone must only be called on an active command: %s", c))
|
||||
}
|
||||
delete(active.cmds, c)
|
||||
}
|
||||
Reference in New Issue
Block a user