adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#307] add package trace, integrate it with logging, and adopt it throughout zrepl

Browse Source 
package trace:

- introduce the concept of tasks and spans, tracked as linked list within ctx
    - see package-level docs for an overview of the concepts
    - **main feature 1**: unique stack of task and span IDs
        - makes it easy to follow a series of log entries in concurrent code
    - **main feature 2**: ability to produce a chrome://tracing-compatible trace file
        - either via an env variable or a `zrepl pprof` subcommand
        - this is not a CPU profile, we already have go pprof for that
        - but it is very useful to visually inspect where the
          replication / snapshotter / pruner spends its time
          ( fixes #307 )

usage in package daemon/logging:

- goal: every log entry should have a trace field with the ID stack from package trace

- make `logging.GetLogger(ctx, Subsys)` the authoritative `logger.Logger` factory function
    - the context carries a linked list of injected fields which
      `logging.GetLogger` adds to the logger it returns
    - `logging.GetLogger` also uses package `trace` to get the
      task-and-span-stack and injects it into the returned logger's fields
This commit is contained in:
Christian Schwarz
2020-04-11 15:49:41 +02:00
parent bcb5965617
commit 10a14a8c50
75 changed files with 1934 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zfs/versions_destroy.go
+2 -2
View File
@@ -38,8 +38,8 @@ func (o *DestroySnapOp) String() string {
return fmt.Sprintf("destroy operation %s@%s", o.Filesystem, o.Name)
}
func ZFSDestroyFilesystemVersions(reqs []*DestroySnapOp) {
doDestroy(context.TODO(), reqs, destroyerSingleton)
func ZFSDestroyFilesystemVersions(ctx context.Context, reqs []*DestroySnapOp) {
doDestroy(ctx, reqs, destroyerSingleton)
}
func setDestroySnapOpErr(b []*DestroySnapOp, err error) {
zfs/zfs_test.go
+1
View File
@@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
)
// FIXME make this a platformtest
func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
t.SkipNow() // FIXME ZFS_BINARY does not work if tests run in parallel
zfs/zfscmd/zfscmd-logging-scraper/zfscmd_logging_scraper.go
+1 -1
View File
@@ -23,7 +23,7 @@ type RuntimeLine struct {
Error string
}
var humanFormatterLineRE = regexp.MustCompile(`^(\[[^\]]+\]){2}\[zfs.cmd\]:\s+command\s+exited\s+(with|without)\s+error\s+(.+)`)
var humanFormatterLineRE = regexp.MustCompile(`^(\[[^\]]+\]){2}\[zfs.cmd\]\[[^\]]+\]:\s+command\s+exited\s+(with|without)\s+error\s+(.+)`)
func parseSecs(s string) (time.Duration, error) {
d, err := time.ParseDuration(s + "s")
zfs/zfscmd/zfscmd-logging-scraper/zfscmd_logging_scrapter_test.go
+3 -3
View File
@@ -30,7 +30,7 @@ func TestParseHumanFormatter(t *testing.T) {
tcs := []testCase{
{
Name: "human-formatter-noerror",
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd]: command exited without error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619"`,
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd][task$stack$span.stack]: command exited without error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619"`,
Expect: &RuntimeLine{
Cmd: "zfs list -H -p -o name -r -t filesystem,volume",
TotalTime: secs("0.037828619"),
@@ -42,7 +42,7 @@ func TestParseHumanFormatter(t *testing.T) {
},
{
Name: "human-formatter-witherror",
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd]: command exited with error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619" err="some error"`,
Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd][task$stack$span.stack]: command exited with error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619" err="some error"`,
Expect: &RuntimeLine{
Cmd: "zfs list -H -p -o name -r -t filesystem,volume",
TotalTime: secs("0.037828619"),
@@ -54,7 +54,7 @@ func TestParseHumanFormatter(t *testing.T) {
},
{
Name: "from graylog",
Input: `2020-04-04T00:00:05+02:00 [DEBG][csnas][zfs.cmd]: command exited without error usertime_s="0" cmd="zfs send -i zroot/ezjail/synapse-12@zrepl_20200329_095518_000 zroot/ezjail/synapse-12@zrepl_20200329_102454_000" total_time_s="0.101598591" invocation="85" systemtime_s="0.041581"`,
Input: `2020-04-04T00:00:05+02:00 [DEBG][csnas][zfs.cmd][task$stack$span.stack]: command exited without error usertime_s="0" cmd="zfs send -i zroot/ezjail/synapse-12@zrepl_20200329_095518_000 zroot/ezjail/synapse-12@zrepl_20200329_102454_000" total_time_s="0.101598591" invocation="85" systemtime_s="0.041581"`,
Expect: &RuntimeLine{
Cmd: "zfs send -i zroot/ezjail/synapse-12@zrepl_20200329_095518_000 zroot/ezjail/synapse-12@zrepl_20200329_102454_000",
TotalTime: secs("0.101598591"),
zfs/zfscmd/zfscmd.go
+15 -8
View File
@@ -14,6 +14,7 @@ import (
"sync"
"time"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/circlog"
)
@@ -22,6 +23,7 @@ type Cmd struct {
ctx context.Context
mtx sync.RWMutex
startedAt, waitStartedAt, waitReturnedAt time.Time
waitReturnEndSpanCb trace.DoneFunc
}
func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
@@ -31,7 +33,7 @@ func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
// err.(*exec.ExitError).Stderr will NOT be set
func (c *Cmd) CombinedOutput() (o []byte, err error) {
c.startPre()
c.startPre(false)
c.startPost(nil)
c.waitPre()
o, err = c.cmd.CombinedOutput()
@@ -41,7 +43,7 @@ func (c *Cmd) CombinedOutput() (o []byte, err error) {
// err.(*exec.ExitError).Stderr will be set
func (c *Cmd) Output() (o []byte, err error) {
c.startPre()
c.startPre(false)
c.startPost(nil)
c.waitPre()
o, err = c.cmd.Output()
@@ -78,7 +80,7 @@ func (c *Cmd) log() Logger {
}
func (c *Cmd) Start() (err error) {
c.startPre()
c.startPre(true)
err = c.cmd.Start()
c.startPost(err)
return err
@@ -95,15 +97,17 @@ func (c *Cmd) Process() *os.Process {
func (c *Cmd) Wait() (err error) {
c.waitPre()
err = c.cmd.Wait()
if !c.waitReturnedAt.IsZero() {
// ignore duplicate waits
return err
}
c.waitPost(err)
return err
}
func (c *Cmd) startPre() {
func (c *Cmd) startPre(newTask bool) {
if newTask {
// avoid explosion of tasks with name c.String()
c.ctx, c.waitReturnEndSpanCb = trace.WithTaskAndSpan(c.ctx, "zfscmd", c.String())
} else {
c.ctx, c.waitReturnEndSpanCb = trace.WithSpan(c.ctx, c.String())
}
startPreLogging(c, time.Now())
}
@@ -178,6 +182,9 @@ func (c *Cmd) waitPost(err error) {
waitPostReport(c, u, now)
waitPostLogging(c, u, err, now)
waitPostPrometheus(c, u, err, now)
// must be last because c.ctx might be used by other waitPost calls
c.waitReturnEndSpanCb()
}
// returns 0 if the command did not yet finish
zfs/zfscmd/zfscmd_context.go
+3 -10
View File
@@ -3,14 +3,14 @@ package zfscmd
import (
"context"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
)
type contextKey int
const (
contextKeyLogger contextKey = iota
contextKeyJobID
contextKeyJobID contextKey = 1 + iota
)
type Logger = logger.Logger
@@ -27,13 +27,6 @@ func getJobIDOrDefault(ctx context.Context, def string) string {
return ret
}
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLogger, log)
}
func getLogger(ctx context.Context) Logger {
if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
return l
}
return logger.NewNullLogger()
return logging.GetLogger(ctx, logging.SubsysZFSCmd)
}
zfs/zfscmd/zfscmd_platform_test.go
+1
View File
@@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/util/circlog"
)