[#307] add package trace, integrate it with logging, and adopt it throughout zrepl

package trace:

- introduce the concept of tasks and spans, tracked as linked list within ctx
    - see package-level docs for an overview of the concepts
    - **main feature 1**: unique stack of task and span IDs
        - makes it easy to follow a series of log entries in concurrent code
    - **main feature 2**: ability to produce a chrome://tracing-compatible trace file
        - either via an env variable or a `zrepl pprof` subcommand
        - this is not a CPU profile, we already have go pprof for that
        - but it is very useful to visually inspect where the
          replication / snapshotter / pruner spends its time
          ( fixes #307 )

usage in package daemon/logging:

- goal: every log entry should have a trace field with the ID stack from package trace

- make `logging.GetLogger(ctx, Subsys)` the authoritative `logger.Logger` factory function
    - the context carries a linked list of injected fields which
      `logging.GetLogger` adds to the logger it returns
    - `logging.GetLogger` also uses package `trace` to get the
      task-and-span-stack and injects it into the returned logger's fields

zfs/zfscmd/zfscmd.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/util/circlog"
 )
 
@@ -22,6 +23,7 @@ type Cmd struct {
 	ctx                                      context.Context
 	mtx                                      sync.RWMutex
 	startedAt, waitStartedAt, waitReturnedAt time.Time
+	waitReturnEndSpanCb                      trace.DoneFunc
 }
 
 func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
@@ -31,7 +33,7 @@ func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
 
 // err.(*exec.ExitError).Stderr will NOT be set
 func (c *Cmd) CombinedOutput() (o []byte, err error) {
-	c.startPre()
+	c.startPre(false)
 	c.startPost(nil)
 	c.waitPre()
 	o, err = c.cmd.CombinedOutput()
@@ -41,7 +43,7 @@ func (c *Cmd) CombinedOutput() (o []byte, err error) {
 
 // err.(*exec.ExitError).Stderr will be set
 func (c *Cmd) Output() (o []byte, err error) {
-	c.startPre()
+	c.startPre(false)
 	c.startPost(nil)
 	c.waitPre()
 	o, err = c.cmd.Output()
@@ -78,7 +80,7 @@ func (c *Cmd) log() Logger {
 }
 
 func (c *Cmd) Start() (err error) {
-	c.startPre()
+	c.startPre(true)
 	err = c.cmd.Start()
 	c.startPost(err)
 	return err
@@ -95,15 +97,17 @@ func (c *Cmd) Process() *os.Process {
 func (c *Cmd) Wait() (err error) {
 	c.waitPre()
 	err = c.cmd.Wait()
-	if !c.waitReturnedAt.IsZero() {
-		// ignore duplicate waits
-		return err
-	}
 	c.waitPost(err)
 	return err
 }
 
-func (c *Cmd) startPre() {
+func (c *Cmd) startPre(newTask bool) {
+	if newTask {
+		// avoid explosion of tasks with name c.String()
+		c.ctx, c.waitReturnEndSpanCb = trace.WithTaskAndSpan(c.ctx, "zfscmd", c.String())
+	} else {
+		c.ctx, c.waitReturnEndSpanCb = trace.WithSpan(c.ctx, c.String())
+	}
 	startPreLogging(c, time.Now())
 }
 
@@ -178,6 +182,9 @@ func (c *Cmd) waitPost(err error) {
 	waitPostReport(c, u, now)
 	waitPostLogging(c, u, err, now)
 	waitPostPrometheus(c, u, err, now)
+
+	// must be last because c.ctx might be used by other waitPost calls
+	c.waitReturnEndSpanCb()
 }
 
 // returns 0 if the command did not yet finish