adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#307] add package trace, integrate it with logging, and adopt it throughout zrepl

Browse Source 
package trace:

- introduce the concept of tasks and spans, tracked as linked list within ctx
    - see package-level docs for an overview of the concepts
    - **main feature 1**: unique stack of task and span IDs
        - makes it easy to follow a series of log entries in concurrent code
    - **main feature 2**: ability to produce a chrome://tracing-compatible trace file
        - either via an env variable or a `zrepl pprof` subcommand
        - this is not a CPU profile, we already have go pprof for that
        - but it is very useful to visually inspect where the
          replication / snapshotter / pruner spends its time
          ( fixes #307 )

usage in package daemon/logging:

- goal: every log entry should have a trace field with the ID stack from package trace

- make `logging.GetLogger(ctx, Subsys)` the authoritative `logger.Logger` factory function
    - the context carries a linked list of injected fields which
      `logging.GetLogger` adds to the logger it returns
    - `logging.GetLogger` also uses package `trace` to get the
      task-and-span-stack and injects it into the returned logger's fields
This commit is contained in:
Christian Schwarz
2020-04-11 15:49:41 +02:00
parent bcb5965617
commit 10a14a8c50
75 changed files with 1934 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/daemon.go
+23 -15
View File
@@ -12,6 +12,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
@@ -23,9 +24,8 @@ import (
"github.com/zrepl/zrepl/zfs/zfscmd"
)
func Run(conf *config.Config) error {
ctx, cancel := context.WithCancel(context.Background())
func Run(ctx context.Context, conf *config.Config) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
sigChan := make(chan os.Signal, 1)
@@ -39,6 +39,7 @@ func Run(conf *config.Config) error {
if err != nil {
return errors.Wrap(err, "cannot build logging from config")
}
outlets.Add(newPrometheusLogOutlet(), logger.Debug)
confJobs, err := job.JobsFromConfig(conf)
if err != nil {
@@ -48,14 +49,23 @@ func Run(conf *config.Config) error {
log := logger.NewLogger(outlets, 1*time.Second)
log.Info(version.NewZreplVersionInformation().String())
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
trace.RegisterCallback(trace.Callback{
OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
logging.
GetLogger(ctx, logging.SubsysTraceData).
WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
},
})
for _, job := range confJobs {
if IsInternalJobName(job.Name()) {
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
}
}
ctx = job.WithLogger(ctx, log)
jobs := newJobs()
// start control socket
@@ -84,6 +94,7 @@ func Run(conf *config.Config) error {
// register global (=non job-local) metrics
zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
trace.RegisterMetrics(prometheus.DefaultRegisterer)
log.Info("starting daemon")
@@ -98,6 +109,8 @@ func Run(conf *config.Config) error {
case <-ctx.Done():
log.WithError(ctx.Err()).Info("context finished")
}
log.Info("waiting for jobs to finish")
<-jobs.wait()
log.Info("daemon exiting")
return nil
}
@@ -120,14 +133,11 @@ func newJobs() *jobs {
}
}
const (
logJobField string = "job"
)
func (s *jobs) wait() <-chan struct{} {
ch := make(chan struct{})
go func() {
s.wg.Wait()
close(ch)
}()
return ch
}
@@ -202,9 +212,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
s.m.Lock()
defer s.m.Unlock()
jobLog := job.GetLogger(ctx).
WithField(logJobField, j.Name()).
WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
jobName := j.Name()
if !internal && IsInternalJobName(jobName) {
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
@@ -219,7 +228,6 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
j.RegisterMetrics(prometheus.DefaultRegisterer)
s.jobs[jobName] = j
ctx = job.WithLogger(ctx, jobLog)
ctx = zfscmd.WithJobID(ctx, j.Name())
ctx, wakeup := wakeup.Context(ctx)
ctx, resetFunc := reset.Context(ctx)
@@ -229,8 +237,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
s.wg.Add(1)
go func() {
defer s.wg.Done()
jobLog.Info("starting job")
defer jobLog.Info("job exited")
job.GetLogger(ctx).Info("starting job")
defer job.GetLogger(ctx).Info("job exited")
j.Run(ctx)
}()
}
daemon/hooks/hook_logging.go
+2 -14
View File
@@ -6,29 +6,17 @@ import (
"context"
"sync"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/util/envconst"
)
type contextKey int
const (
contextKeyLog contextKey = 0
)
type Logger = logger.Logger
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, log)
}
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
func getLogger(ctx context.Context) Logger {
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
return log
}
return logger.NewNullLogger()
return logging.GetLogger(ctx, logging.SubsysHooks)
}
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
daemon/hooks/hooks_test.go
+6 -2
View File
@@ -10,9 +10,11 @@ import (
"text/template"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs"
)
@@ -69,6 +71,9 @@ func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret val
}
func TestHooks(t *testing.T) {
ctx, end := trace.WithTaskFromStack(context.Background())
defer end()
testFSName := "testpool/testdataset"
testSnapshotName := "testsnap"
@@ -418,9 +423,8 @@ jobs:
cbReached = false
ctx := context.Background()
if testing.Verbose() && !tt.SuppressOutput {
ctx = hooks.WithLogger(ctx, log)
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
}
plan.Run(ctx, false)
report := plan.Report()
daemon/job/active.go
+30 -22
View File
@@ -8,12 +8,13 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -79,7 +80,7 @@ func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
}
type activeMode interface {
ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
DisconnectEndpoints()
SenderReceiver() (logic.Sender, logic.Receiver)
Type() Type
@@ -98,14 +99,14 @@ type modePush struct {
snapper *snapper.PeriodicOrManual
}
func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
m.setupMtx.Lock()
defer m.setupMtx.Unlock()
if m.receiver != nil || m.sender != nil {
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
}
m.sender = endpoint.NewSender(*m.senderConfig)
m.receiver = rpc.NewClient(connecter, loggers)
m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
}
func (m *modePush) DisconnectEndpoints() {
@@ -176,14 +177,14 @@ type modePull struct {
interval config.PositiveDurationOrManual
}
func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
m.setupMtx.Lock()
defer m.setupMtx.Unlock()
if m.receiver != nil || m.sender != nil {
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
}
m.receiver = endpoint.NewReceiver(m.receiverConfig)
m.sender = rpc.NewClient(connecter, loggers)
m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
}
func (m *modePull) DisconnectEndpoints() {
@@ -376,15 +377,18 @@ func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
}
func (j *ActiveSide) Run(ctx context.Context) {
ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
defer endTask()
log := GetLogger(ctx)
ctx = logging.WithSubsystemLoggers(ctx, log)
defer log.Info("job exiting")
periodicDone := make(chan struct{})
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go j.mode.RunPeriodic(ctx, periodicDone)
periodicCtx, endTask := trace.WithTask(ctx, "periodic")
defer endTask()
go j.mode.RunPeriodic(periodicCtx, periodicDone)
invocationCount := 0
outer:
@@ -400,17 +404,15 @@ outer:
case <-periodicDone:
}
invocationCount++
invLog := log.WithField("invocation", invocationCount)
j.do(WithLogger(ctx, invLog))
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
j.do(invocationCtx)
endSpan()
}
}
func (j *ActiveSide) do(ctx context.Context) {
log := GetLogger(ctx)
ctx = logging.WithSubsystemLoggers(ctx, log)
loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
j.mode.ConnectEndpoints(loggers, j.connecter)
j.mode.ConnectEndpoints(ctx, j.connecter)
defer j.mode.DisconnectEndpoints()
// allow cancellation of an invocation (this function)
@@ -433,20 +435,22 @@ func (j *ActiveSide) do(ctx context.Context) {
return
default:
}
ctx, endSpan := trace.WithSpan(ctx, "replication")
ctx, repCancel := context.WithCancel(ctx)
var repWait driver.WaitFunc
j.updateTasks(func(tasks *activeSideTasks) {
// reset it
*tasks = activeSideTasks{}
tasks.replicationCancel = repCancel
tasks.replicationCancel = func() { repCancel(); endSpan() }
tasks.replicationReport, repWait = replication.Do(
ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
)
tasks.state = ActiveSideReplicating
})
log.Info("start replication")
GetLogger(ctx).Info("start replication")
repWait(true) // wait blocking
repCancel() // always cancel to free up context resources
endSpan()
}
{
@@ -455,16 +459,18 @@ func (j *ActiveSide) do(ctx context.Context) {
return
default:
}
ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
ctx, senderCancel := context.WithCancel(ctx)
tasks := j.updateTasks(func(tasks *activeSideTasks) {
tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
tasks.prunerSenderCancel = senderCancel
tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
tasks.state = ActiveSidePruneSender
})
log.Info("start pruning sender")
GetLogger(ctx).Info("start pruning sender")
tasks.prunerSender.Prune()
log.Info("finished pruning sender")
GetLogger(ctx).Info("finished pruning sender")
senderCancel()
endSpan()
}
{
select {
@@ -472,16 +478,18 @@ func (j *ActiveSide) do(ctx context.Context) {
return
default:
}
ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
ctx, receiverCancel := context.WithCancel(ctx)
tasks := j.updateTasks(func(tasks *activeSideTasks) {
tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
tasks.prunerReceiverCancel = receiverCancel
tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
tasks.state = ActiveSidePruneReceiver
})
log.Info("start pruning receiver")
GetLogger(ctx).Info("start pruning receiver")
tasks.prunerReceiver.Prune()
log.Info("finished pruning receiver")
GetLogger(ctx).Info("finished pruning receiver")
receiverCancel()
endSpan()
}
j.updateTasks(func(tasks *activeSideTasks) {
daemon/job/job.go
+2 -14
View File
@@ -7,6 +7,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/zfs"
@@ -14,21 +15,8 @@ import (
type Logger = logger.Logger
type contextKey int
const (
contextKeyLog contextKey = iota
)
func GetLogger(ctx context.Context) Logger {
if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
return l
}
return logger.NewNullLogger()
}
func WithLogger(ctx context.Context, l Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, l)
return logging.GetLogger(ctx, logging.SubsysJob)
}
type Job interface {
daemon/job/passive.go
+14 -5
View File
@@ -6,6 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
@@ -164,12 +165,14 @@ func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
func (j *PassiveSide) Run(ctx context.Context) {
ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
defer endTask()
log := GetLogger(ctx)
defer log.Info("job exiting")
ctx = logging.WithSubsystemLoggers(ctx, log)
{
ctx, cancel := context.WithCancel(ctx) // shadowing
ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
defer endTask()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go j.mode.RunPeriodic(ctx)
}
@@ -179,8 +182,14 @@ func (j *PassiveSide) Run(ctx context.Context) {
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
}
ctxInterceptor := func(handlerCtx context.Context) context.Context {
return logging.WithSubsystemLoggers(handlerCtx, log)
ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
// the handlerCtx is clean => need to inherit logging and tracing config from job context
handlerCtx = logging.WithInherit(handlerCtx, ctx)
handlerCtx = trace.WithInherit(handlerCtx, ctx)
handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
defer endTask()
handler(handlerCtx)
}
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
daemon/job/snapjob.go
+13 -6
View File
@@ -2,15 +2,16 @@ package job
import (
"context"
"fmt"
"sort"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -89,15 +90,18 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
func (j *SnapJob) Run(ctx context.Context) {
ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
defer endTask()
log := GetLogger(ctx)
ctx = logging.WithSubsystemLoggers(ctx, log)
defer log.Info("job exiting")
periodicDone := make(chan struct{})
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go j.snapper.Run(ctx, periodicDone)
periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
defer endTask()
go j.snapper.Run(periodicCtx, periodicDone)
invocationCount := 0
outer:
@@ -112,8 +116,10 @@ outer:
case <-periodicDone:
}
invocationCount++
invLog := log.WithField("invocation", invocationCount)
j.doPrune(WithLogger(ctx, invLog))
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
j.doPrune(invocationCtx)
endSpan()
}
}
@@ -161,8 +167,9 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
}
func (j *SnapJob) doPrune(ctx context.Context) {
ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
defer endSpan()
log := GetLogger(ctx)
ctx = logging.WithSubsystemLoggers(ctx, log)
sender := endpoint.NewSender(endpoint.SenderConfig{
JobID: j.name,
FSF: j.fsfilter,
daemon/logging/build_logging.go
+96 -31
View File
@@ -9,20 +9,10 @@ import (
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/replication/driver"
"github.com/zrepl/zrepl/replication/logic"
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/rpc/transportmux"
"github.com/zrepl/zrepl/tlsconf"
"github.com/zrepl/zrepl/transport"
"github.com/zrepl/zrepl/zfs/zfscmd"
)
func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
@@ -70,6 +60,8 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
type Subsystem string
const (
SubsysMeta Subsystem = "meta"
SubsysJob Subsystem = "job"
SubsysReplication Subsystem = "repl"
SubsysEndpoint Subsystem = "endpoint"
SubsysPruning Subsystem = "pruning"
@@ -81,30 +73,103 @@ const (
SubsysRPCControl Subsystem = "rpc.ctrl"
SubsysRPCData Subsystem = "rpc.data"
SubsysZFSCmd Subsystem = "zfs.cmd"
SubsysTraceData Subsystem = "trace.data"
SubsysPlatformtest Subsystem = "platformtest"
)
func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsysEndpoint))
ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
ctx = zfscmd.WithLogger(ctx, log.WithField(SubsysField, SubsysZFSCmd))
ctx = rpc.WithLoggers(ctx,
rpc.Loggers{
General: log.WithField(SubsysField, SubsysRPC),
Control: log.WithField(SubsysField, SubsysRPCControl),
Data: log.WithField(SubsysField, SubsysRPCData),
},
)
return ctx
var AllSubsystems = []Subsystem{
SubsysMeta,
SubsysJob,
SubsysReplication,
SubsysEndpoint,
SubsysPruning,
SubsysSnapshot,
SubsysHooks,
SubsysTransport,
SubsysTransportMux,
SubsysRPC,
SubsysRPCControl,
SubsysRPCData,
SubsysZFSCmd,
SubsysTraceData,
SubsysPlatformtest,
}
func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
return log.ReplaceField(SubsysField, subsys)
type injectedField struct {
field string
value interface{}
parent *injectedField
}
func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
var parent *injectedField
parentI := ctx.Value(contextKeyInjectedField)
if parentI != nil {
parent = parentI.(*injectedField)
}
// TODO sanity-check `field` now
this := &injectedField{field, value, parent}
return context.WithValue(ctx, contextKeyInjectedField, this)
}
func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
injI := ctx.Value(contextKeyInjectedField)
if injI == nil {
return
}
inj := injI.(*injectedField)
for ; inj != nil; inj = inj.parent {
cb(inj.field, inj.value)
}
}
type SubsystemLoggers map[Subsystem]logger.Logger
func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
loggers := make(SubsystemLoggers)
for _, s := range AllSubsystems {
loggers[s] = l
}
return loggers
}
func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
return context.WithValue(ctx, contextKeyLoggers, loggers)
}
func GetLoggers(ctx context.Context) SubsystemLoggers {
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
if !ok {
return nil
}
return loggers
}
func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
return getLoggerImpl(ctx, subsys, true)
}
func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
if !ok || loggers == nil {
return logger.NewNullLogger()
}
l, ok := loggers[subsys]
if !ok {
return logger.NewNullLogger()
}
l = l.WithField(SubsysField, subsys)
l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
fields := make(logger.Fields)
iterInjectedFields(ctx, func(field string, value interface{}) {
fields[field] = value
})
l = l.WithFields(fields)
return l
}
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
daemon/logging/logging_context.go
+24
View File
@@ -0,0 +1,24 @@
package logging
import "context"
type contextKey int
const (
contextKeyLoggers contextKey = 1 + iota
contextKeyInjectedField
)
var contextKeys = []contextKey{
contextKeyLoggers,
contextKeyInjectedField,
}
func WithInherit(ctx, inheritFrom context.Context) context.Context {
for _, k := range contextKeys {
if v := inheritFrom.Value(k); v != nil {
ctx = context.WithValue(ctx, k, v) // no shadow
}
}
return ctx
}
daemon/logging/logging_formatters.go
+4 -3
View File
@@ -22,6 +22,7 @@ const (
const (
JobField string = "job"
SubsysField string = "subsystem"
SpanField string = "span"
)
type MetadataFlags int64
@@ -85,7 +86,7 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
}
prefixFields := []string{JobField, SubsysField}
prefixFields := []string{JobField, SubsysField, SpanField}
prefixed := make(map[string]bool, len(prefixFields)+2)
for _, field := range prefixFields {
val, ok := e.Fields[field]
@@ -174,8 +175,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
}
// at least try and put job and task in front
prefixed := make(map[string]bool, 2)
prefix := []string{JobField, SubsysField}
prefixed := make(map[string]bool, 3)
prefix := []string{JobField, SubsysField, SpanField}
for _, pf := range prefix {
v, ok := e.Fields[pf]
if !ok {
daemon/logging/trace/trace.go
+386
View File
@@ -0,0 +1,386 @@
// package trace provides activity tracing via ctx through Tasks and Spans
//
// Basic Concepts
//
// Tracing can be used to identify where a piece of code spends its time.
//
// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
// to understand what happens inside the Go runtime.
// However, it is not ideal for application level tracing, in particular if those traces should be understandable
// to tech-savvy users (albeit not developers).
//
// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
//
// - Neither task nor span is really tangible but instead contained within the context.Context tree
// - Tasks represent concurrent activity (i.e. goroutines).
// - Spans represent a semantic stack trace within a task.
//
// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
//
// go func(ctx context.Context) {
// ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
// defer endTask()
// // ...
// }(ctx)
//
// Within the task, you can open up a hierarchy of spans.
// In contrast to tasks, which have can multiple concurrently running child tasks,
// spans must nest and not cross the goroutine boundary.
//
// ctx, endSpan = WithSpan(ctx, "copy-dir")
// defer endSpan()
// for _, f := range dir.Files() {
// func() {
// ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
// defer endspan()
// b, _ := ioutil.ReadFile(f)
// _ = ioutil.WriteFile(f + ".copy", b, 0600)
// }()
// }
//
// In combination:
// ctx, endTask = WithTask(ctx, "copy-dirs")
// defer endTask()
// for i := range dirs {
// go func(dir string) {
// ctx, endTask := WithTask(ctx, "copy-dir")
// defer endTask()
// for _, f := range filesIn(dir) {
// func() {
// ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
// defer endspan()
// b, _ := ioutil.ReadFile(f)
// _ = ioutil.WriteFile(f + ".copy", b, 0600)
// }()
// }
// }()
// }
//
// Note that a span ends at the time you call endSpan - not before and not after that.
// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
// the out-of-order endSpan() will panic.
//
// A similar rule applies to the endTask closure returned by WithTask:
// If a task has live child tasks at the time you call endTask(), the call will panic.
//
// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
//
//
// Best Practices For Naming Tasks And Spans
//
// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
// Also, the package appends `#NUM` for each concurrently running instance of a task name.
// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
// infinite number of horizontal bars in the visualization.
//
//
// Chrome-compatible Tracefile Support
//
// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
// that can be loaded into chrome://tracing .
// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
//
// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
// From process start onward, a trace is written to that path.
//
// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
//
// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
// will not receive further trace output.
package trace
import (
"context"
"fmt"
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/util/chainlock"
)
var metrics struct {
activeTasks prometheus.Gauge
}
var taskNamer *uniqueConcurrentTaskNamer = newUniqueTaskNamer()
func init() {
metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "zrepl",
Subsystem: "trace",
Name: "active_tasks",
Help: "number of active (tracing-level) tasks in the daemon",
})
}
func RegisterMetrics(r prometheus.Registerer) {
r.MustRegister(metrics.activeTasks)
}
type traceNode struct {
id string
annotation string
parentTask *traceNode
mtx chainlock.L
activeChildTasks int32 // only for task nodes, insignificant for span nodes
parentSpan *traceNode
activeChildSpan *traceNode // nil if task or span doesn't have an active child span
startedAt time.Time
endedAt time.Time
}
func (s *traceNode) StartedAt() time.Time { return s.startedAt }
func (s *traceNode) EndedAt() time.Time { return s.endedAt }
// Returned from WithTask or WithSpan.
// Must be called once the task or span ends.
// See package-level docs for nesting rules.
// Wrong call order / forgetting to call it will result in panics.
type DoneFunc func()
var ErrTaskStillHasActiveChildTasks = fmt.Errorf("end task: task still has active child tasks")
// Start a new root task or create a child task of an existing task.
//
// This is required when starting a new goroutine and
// passing an existing task context to it.
//
// taskName should be a constantand must not contain '#'
//
// The implementation ensures that,
// if multiple tasks with the same name exist simultaneously,
// a unique suffix is appended to uniquely identify the task opened with this function.
func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
var parentTask *traceNode
nodeI := ctx.Value(contextKeyTraceNode)
if nodeI != nil {
node := nodeI.(*traceNode)
if node.parentSpan != nil {
parentTask = node.parentTask
} else {
parentTask = node
}
}
// find the first ancestor that hasn't ended yet (nil if need be)
if parentTask != nil {
parentTask.mtx.Lock()
}
for parentTask != nil && !parentTask.endedAt.IsZero() {
thisParent := parentTask
parentTask = parentTask.parentTask
// lock parent first such that it isn't modified by other callers to thisParent
if parentTask != nil {
parentTask.mtx.Lock()
}
thisParent.mtx.Unlock()
}
// invariant: either parentTask != nil and we hold the lock on parentTask, or parentTask is nil
taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
this := &traceNode{
id: genID(),
annotation: taskName,
parentTask: parentTask,
activeChildTasks: 0,
parentSpan: nil,
activeChildSpan: nil,
startedAt: time.Now(),
endedAt: time.Time{},
}
if parentTask != nil {
this.parentTask.activeChildTasks++
parentTask.mtx.Unlock()
}
ctx = context.WithValue(ctx, contextKeyTraceNode, this)
chrometraceBeginTask(this)
metrics.activeTasks.Inc()
endTaskFunc := func() {
// only hold locks while manipulating the tree
// (trace writer might block too long and unlike spans, tasks are updated concurrently)
alreadyEnded := func() (alreadyEnded bool) {
if this.parentTask != nil {
defer this.parentTask.mtx.Lock().Unlock()
}
defer this.mtx.Lock().Unlock()
if this.activeChildTasks != 0 {
panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildTasks))
}
// support idempotent task ends
if !this.endedAt.IsZero() {
return true
}
this.endedAt = time.Now()
if this.parentTask != nil {
this.parentTask.activeChildTasks--
if this.parentTask.activeChildTasks < 0 {
panic("impl error: parent task with negative activeChildTasks count")
}
}
return false
}()
if alreadyEnded {
return
}
chrometraceEndTask(this)
metrics.activeTasks.Dec()
taskNameDone()
}
return ctx, endTaskFunc
}
var ErrAlreadyActiveChildSpan = fmt.Errorf("create child span: span already has an active child span")
var ErrSpanStillHasActiveChildSpan = fmt.Errorf("end span: span still has active child spans")
// Start a new span.
// Important: ctx must have an active task (see WithTask)
func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
var parentSpan, parentTask *traceNode
nodeI := ctx.Value(contextKeyTraceNode)
if nodeI != nil {
parentSpan = nodeI.(*traceNode)
if parentSpan.parentSpan == nil {
parentTask = parentSpan
} else {
parentTask = parentSpan.parentTask
}
} else {
panic("must be called from within a task")
}
this := &traceNode{
id: genID(),
annotation: annotation,
parentTask: parentTask,
parentSpan: parentSpan,
activeChildSpan: nil,
startedAt: time.Now(),
endedAt: time.Time{},
}
parentSpan.mtx.HoldWhile(func() {
if parentSpan.activeChildSpan != nil {
panic(ErrAlreadyActiveChildSpan)
}
parentSpan.activeChildSpan = this
})
ctx = context.WithValue(ctx, contextKeyTraceNode, this)
chrometraceBeginSpan(this)
callbackEndSpan := callbackBeginSpan(ctx)
endTaskFunc := func() {
defer parentSpan.mtx.Lock().Unlock()
if parentSpan.activeChildSpan != this && this.endedAt.IsZero() {
panic("impl error: activeChildSpan should not change while != nil because there can only be one")
}
defer this.mtx.Lock().Unlock()
if this.activeChildSpan != nil {
panic(ErrSpanStillHasActiveChildSpan)
}
if !this.endedAt.IsZero() {
return // support idempotent span ends
}
parentSpan.activeChildSpan = nil
this.endedAt = time.Now()
chrometraceEndSpan(this)
callbackEndSpan(this)
}
return ctx, endTaskFunc
}
type StackKind struct {
symbolizeTask func(t *traceNode) string
symbolizeSpan func(s *traceNode) string
}
var (
StackKindId = &StackKind{
symbolizeTask: func(t *traceNode) string { return t.id },
symbolizeSpan: func(s *traceNode) string { return s.id },
}
SpanStackKindCombined = &StackKind{
symbolizeTask: func(t *traceNode) string { return fmt.Sprintf("(%s %q)", t.id, t.annotation) },
symbolizeSpan: func(s *traceNode) string { return fmt.Sprintf("(%s %q)", s.id, s.annotation) },
}
SpanStackKindAnnotation = &StackKind{
symbolizeTask: func(t *traceNode) string { return t.annotation },
symbolizeSpan: func(s *traceNode) string { return s.annotation },
}
)
func (n *traceNode) task() *traceNode {
task := n.parentTask
if n.parentSpan == nil {
task = n
}
return task
}
func (n *traceNode) TaskName() string {
task := n.task()
return task.annotation
}
func (this *traceNode) TaskAndSpanStack(kind *StackKind) (spanIdStack string) {
task := this.task()
var spansInTask []*traceNode
for s := this; s != nil; s = s.parentSpan {
spansInTask = append(spansInTask, s)
}
var tasks []*traceNode
for t := task; t != nil; t = t.parentTask {
tasks = append(tasks, t)
}
var taskIdsRev []string
for i := len(tasks) - 1; i >= 0; i-- {
taskIdsRev = append(taskIdsRev, kind.symbolizeTask(tasks[i]))
}
var spanIdsRev []string
for i := len(spansInTask) - 1; i >= 0; i-- {
spanIdsRev = append(spanIdsRev, kind.symbolizeSpan(spansInTask[i]))
}
taskStack := strings.Join(taskIdsRev, "$")
return fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
}
func GetSpanStackOrDefault(ctx context.Context, kind StackKind, def string) string {
if nI := ctx.Value(contextKeyTraceNode); nI != nil {
n := nI.(*traceNode)
return n.TaskAndSpanStack(StackKindId)
} else {
return def
}
}
daemon/logging/trace/trace_callback.go
+54
View File
@@ -0,0 +1,54 @@
package trace
import (
"context"
"time"
"github.com/zrepl/zrepl/util/chainlock"
)
type SpanInfo interface {
StartedAt() time.Time
EndedAt() time.Time
TaskAndSpanStack(kind *StackKind) string
}
type Callback struct {
OnBegin func(ctx context.Context)
OnEnd func(ctx context.Context, spanInfo SpanInfo)
}
var callbacks struct {
mtx chainlock.L
cs []Callback
}
func RegisterCallback(c Callback) {
callbacks.mtx.HoldWhile(func() {
callbacks.cs = append(callbacks.cs, c)
})
}
func callbackBeginSpan(ctx context.Context) func(SpanInfo) {
// capture the current state of callbacks into a local variable
// this is safe because the slice is append-only and immutable
// (it is important that a callback registered _after_ callbackBeginSpin is called does not get called on OnEnd)
var cbs []Callback
callbacks.mtx.HoldWhile(func() {
cbs = callbacks.cs
})
for _, cb := range cbs {
if cb.OnBegin != nil {
cb.OnBegin(ctx)
}
}
return func(spanInfo SpanInfo) {
for _, cb := range cbs {
if cb.OnEnd != nil {
cb.OnEnd(ctx, spanInfo)
}
}
}
}
daemon/logging/trace/trace_chrometrace.go
+230
View File
@@ -0,0 +1,230 @@
package trace
// The functions in this file are concerned with the generation
// of trace files based on the information from WithTask and WithSpan.
//
// The emitted trace files are open-ended array of JSON objects
// that follow the Chrome trace file format:
// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
//
// The emitted JSON can be loaded into Chrome's chrome://tracing view.
//
// The trace file can be written to a file whose path is specified in an env file,
// and be written to web sockets established on ChrometraceHttpHandler
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"sync"
"sync/atomic"
"golang.org/x/net/websocket"
"github.com/zrepl/zrepl/util/envconst"
)
var chrometracePID string
func init() {
var err error
chrometracePID, err = os.Hostname()
if err != nil {
panic(err)
}
chrometracePID = fmt.Sprintf("%q", chrometracePID)
}
type chrometraceEvent struct {
Cat string `json:"cat,omitempty"`
Name string `json:"name"`
Stack []string `json:"stack,omitempty"`
Phase string `json:"ph"`
TimestampUnixMicroseconds int64 `json:"ts"`
DurationMicroseconds int64 `json:"dur,omitempty"`
Pid string `json:"pid"`
Tid string `json:"tid"`
Id string `json:"id,omitempty"`
}
func chrometraceBeginSpan(s *traceNode) {
taskName := s.TaskName()
chrometraceWrite(chrometraceEvent{
Name: s.annotation,
Phase: "B",
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
Pid: chrometracePID,
Tid: taskName,
})
}
func chrometraceEndSpan(s *traceNode) {
taskName := s.TaskName()
chrometraceWrite(chrometraceEvent{
Name: s.annotation,
Phase: "E",
TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
Pid: chrometracePID,
Tid: taskName,
})
}
var chrometraceFlowId uint64
func chrometraceBeginTask(s *traceNode) {
chrometraceBeginSpan(s)
if s.parentTask == nil {
return
}
// beginning of a task that has a parent
// => use flow events to link parent and child
flowId := atomic.AddUint64(&chrometraceFlowId, 1)
flowIdStr := fmt.Sprintf("%x", flowId)
parentTask := s.parentTask.TaskName()
chrometraceWrite(chrometraceEvent{
Cat: "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
Name: "child-task",
Phase: "s",
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
Pid: chrometracePID,
Tid: parentTask,
Id: flowIdStr,
})
childTask := s.TaskName()
if parentTask == childTask {
panic(parentTask)
}
chrometraceWrite(chrometraceEvent{
Cat: "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
Name: "child-task",
Phase: "f",
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
Pid: chrometracePID,
Tid: childTask,
Id: flowIdStr,
})
}
func chrometraceEndTask(s *traceNode) {
chrometraceEndSpan(s)
}
type chrometraceConsumerRegistration struct {
w io.Writer
// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
errored chan error
}
var chrometraceConsumers struct {
register chan chrometraceConsumerRegistration
consumers map[chrometraceConsumerRegistration]bool
write chan []byte
}
func init() {
chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
chrometraceConsumers.write = make(chan []byte)
go func() {
kickConsumer := func(c chrometraceConsumerRegistration, err error) {
debug("chrometrace kicking consumer %#v after error %v", c, err)
select {
case c.errored <- err:
default:
}
close(c.errored)
delete(chrometraceConsumers.consumers, c)
}
for {
select {
case reg := <-chrometraceConsumers.register:
debug("registered chrometrace consumer %#v", reg)
chrometraceConsumers.consumers[reg] = true
n, err := reg.w.Write([]byte("[\n"))
if err != nil {
kickConsumer(reg, err)
} else if n != 2 {
kickConsumer(reg, fmt.Errorf("short write: %v", n))
}
// successfully registered
case buf := <-chrometraceConsumers.write:
debug("chrometrace write request: %s", string(buf))
var r bytes.Reader
for c := range chrometraceConsumers.consumers {
r.Reset(buf)
n, err := io.Copy(c.w, &r)
debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
if err != nil {
kickConsumer(c, err)
}
}
}
}
}()
}
func chrometraceWrite(i interface{}) {
var buf bytes.Buffer
err := json.NewEncoder(&buf).Encode(i)
if err != nil {
panic(err)
}
buf.WriteString(",")
chrometraceConsumers.write <- buf.Bytes()
}
func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
defer conn.Close()
var wg sync.WaitGroup
defer wg.Wait()
wg.Add(1)
go func() {
defer wg.Done()
r := bufio.NewReader(conn)
_, _, _ = r.ReadLine() // ignore errors
conn.Close()
}()
errored := make(chan error, 1)
chrometraceConsumers.register <- chrometraceConsumerRegistration{
w: conn,
errored: errored,
}
wg.Add(1)
go func() {
defer wg.Done()
<-errored
conn.Close()
}()
}
var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
func init() {
if chrometraceFileConsumerPath != "" {
var err error
f, err := os.Create(chrometraceFileConsumerPath)
if err != nil {
panic(err)
}
errored := make(chan error, 1)
chrometraceConsumers.register <- chrometraceConsumerRegistration{
w: f,
errored: errored,
}
go func() {
<-errored
f.Close()
}()
}
}
daemon/logging/trace/trace_context.go
+27
View File
@@ -0,0 +1,27 @@
package trace
import "context"
type contextKey int
const (
contextKeyTraceNode contextKey = 1 + iota
)
var contextKeys = []contextKey{
contextKeyTraceNode,
}
// WithInherit inherits the task hierarchy from inheritFrom into ctx.
// The returned context is a child of ctx, but its task and span are those of inheritFrom.
//
// Note that in most use cases, callers most likely want to call WithTask since it will most likely
// be in some sort of connection handler context.
func WithInherit(ctx, inheritFrom context.Context) context.Context {
for _, k := range contextKeys {
if v := inheritFrom.Value(k); v != nil {
ctx = context.WithValue(ctx, k, v) // no shadow
}
}
return ctx
}
daemon/logging/trace/trace_convenience.go
+74
View File
@@ -0,0 +1,74 @@
package trace
import (
"context"
"fmt"
"runtime"
"strings"
"sync"
)
// use like this:
//
// defer WithSpanFromStackUpdateCtx(&existingCtx)()
//
//
func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
*ctx = childSpanCtx
return end
}
// derive task name from call stack (caller's name)
func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
return WithTask(ctx, getMyCallerOrPanic())
}
// derive task name from call stack (caller's name) and update *ctx
// to point to be the child task ctx
func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
child, end := WithTask(*ctx, getMyCallerOrPanic())
*ctx = child
return end
}
// create a task and a span within it in one call
func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
ctx, endTask := WithTask(ctx, task)
ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
return ctx, func() {
endSpan()
endTask()
}
}
// create a span during which several child tasks are spawned using the `add` function
func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
var wg sync.WaitGroup
ctx, endSpan := WithSpan(ctx, taskGroup)
add = func(f func(context.Context)) {
wg.Add(1)
defer wg.Done()
ctx, endTask := WithTask(ctx, taskGroup)
defer endTask()
f(ctx)
}
waitEnd = func() {
wg.Wait()
endSpan()
}
return ctx, add, waitEnd
}
func getMyCallerOrPanic() string {
pc, _, _, ok := runtime.Caller(2)
if !ok {
panic("cannot get caller")
}
details := runtime.FuncForPC(pc)
if ok && details != nil {
const prefix = "github.com/zrepl/zrepl"
return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
}
return ""
}
daemon/logging/trace/trace_convenience_test.go
+16
View File
@@ -0,0 +1,16 @@
package trace
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetCallerOrPanic(t *testing.T) {
withStackFromCtxMock := func() string {
return getMyCallerOrPanic()
}
ret := withStackFromCtxMock()
// zrepl prefix is stripped
assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
}
daemon/logging/trace/trace_debug.go
+15
View File
@@ -0,0 +1,15 @@
package trace
import (
"fmt"
"os"
)
const debugEnabled = false
func debug(format string, args ...interface{}) {
if !debugEnabled {
return
}
fmt.Fprintf(os.Stderr, format+"\n", args...)
}
daemon/logging/trace/trace_genID.go
+47
View File
@@ -0,0 +1,47 @@
package trace
import (
"encoding/base64"
"math/rand"
"os"
"strings"
"time"
"github.com/zrepl/zrepl/util/envconst"
)
var genIdPRNG = rand.New(rand.NewSource(1))
func init() {
genIdPRNG.Seed(time.Now().UnixNano())
genIdPRNG.Seed(int64(os.Getpid()))
}
var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
func init() {
if genIdNumBytes < 1 {
panic("trace node id byte length must be at least 1")
}
}
func genID() string {
var out strings.Builder
enc := base64.NewEncoder(base64.RawStdEncoding, &out)
buf := make([]byte, genIdNumBytes)
for i := 0; i < len(buf); {
n, err := genIdPRNG.Read(buf[i:])
if err != nil {
panic(err)
}
i += n
}
n, err := enc.Write(buf[:])
if err != nil || n != len(buf) {
panic(err)
}
if err := enc.Close(); err != nil {
panic(err)
}
return out.String()
}
daemon/logging/trace/trace_test.go
+205
View File
@@ -0,0 +1,205 @@
package trace
import (
"context"
"fmt"
"testing"
"github.com/gitchander/permutation"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRegularSpanUsage(t *testing.T) {
root, endRoot := WithTask(context.Background(), "root")
defer endRoot()
s1, endS1 := WithSpan(root, "parent")
s2, endS2 := WithSpan(s1, "child")
_, endS3 := WithSpan(s2, "grand-child")
require.NotPanics(t, func() { endS3() })
require.NotPanics(t, func() { endS2() })
// reuse
_, endS4 := WithSpan(s1, "child-2")
require.NotPanics(t, func() { endS4() })
// close parent
require.NotPanics(t, func() { endS1() })
}
func TestMultipleActiveChildSpansNotAllowed(t *testing.T) {
root, endRoot := WithTask(context.Background(), "root")
defer endRoot()
s1, _ := WithSpan(root, "s1")
_, endS2 := WithSpan(s1, "s1-child1")
require.PanicsWithValue(t, ErrAlreadyActiveChildSpan, func() {
_, _ = WithSpan(s1, "s1-child2")
})
endS2()
require.NotPanics(t, func() {
_, _ = WithSpan(s1, "s1-child2")
})
}
func TestForkingChildSpansNotAllowed(t *testing.T) {
root, endRoot := WithTask(context.Background(), "root")
defer endRoot()
s1, _ := WithSpan(root, "s1")
sc, endSC := WithSpan(s1, "s1-child")
_, _ = WithSpan(sc, "s1-child-child")
require.PanicsWithValue(t, ErrSpanStillHasActiveChildSpan, func() {
endSC()
})
}
func TestRegularTaskUsage(t *testing.T) {
// assert concurrent activities on different tasks can end in any order
closeOrder := []int{0, 1, 2}
closeOrders := permutation.New(permutation.IntSlice(closeOrder))
for closeOrders.Next() {
t.Run(fmt.Sprintf("%v", closeOrder), func(t *testing.T) {
root, endRoot := WithTask(context.Background(), "root")
defer endRoot()
c1, endC1 := WithTask(root, "c1")
defer endC1()
c2, endC2 := WithTask(root, "c2")
defer endC2()
// begin 3 concurrent activities
_, endAR := WithSpan(root, "aR")
_, endAC1 := WithSpan(c1, "aC1")
_, endAC2 := WithSpan(c2, "aC2")
endFuncs := []DoneFunc{endAR, endAC1, endAC2}
for _, i := range closeOrder {
require.NotPanics(t, func() {
endFuncs[i]()
}, "%v", i)
}
})
}
}
func TestTaskEndWithActiveChildTaskNotAllowed(t *testing.T) {
root, _ := WithTask(context.Background(), "root")
c, endC := WithTask(root, "child")
_, _ = WithTask(c, "grand-child")
func() {
defer func() {
r := recover()
require.NotNil(t, r)
err, ok := r.(error)
require.True(t, ok)
require.Equal(t, ErrTaskStillHasActiveChildTasks, errors.Cause(err))
}()
endC()
}()
}
func TestIdempotentEndTask(t *testing.T) {
_, end := WithTask(context.Background(), "root")
end()
require.NotPanics(t, func() { end() })
}
func TestSpansPanicIfNoParentTask(t *testing.T) {
require.Panics(t, func() { WithSpan(context.Background(), "taskless-span") })
}
func TestIdempotentEndSpan(t *testing.T) {
root, _ := WithTask(context.Background(), "root")
_, end := WithSpan(root, "span")
end()
require.NotPanics(t, func() { end() })
}
func logAndGetTraceNode(t *testing.T, descr string, ctx context.Context) *traceNode {
n, ok := ctx.Value(contextKeyTraceNode).(*traceNode)
require.True(t, ok)
t.Logf("% 20s %p %#v", descr, n, n)
return n
}
func TestWhiteboxHierachy(t *testing.T) {
root, e1 := WithTask(context.Background(), "root")
rootN := logAndGetTraceNode(t, "root", root)
assert.Nil(t, rootN.parentTask)
assert.Nil(t, rootN.parentSpan)
child, e2 := WithSpan(root, "child")
childN := logAndGetTraceNode(t, "child", child)
assert.Equal(t, rootN, childN.parentTask)
assert.Equal(t, rootN, childN.parentSpan)
grandchild, e3 := WithSpan(child, "grandchild")
grandchildN := logAndGetTraceNode(t, "grandchild", grandchild)
assert.Equal(t, rootN, grandchildN.parentTask)
assert.Equal(t, childN, grandchildN.parentSpan)
gcTask, e4 := WithTask(grandchild, "grandchild-task")
gcTaskN := logAndGetTraceNode(t, "grandchild-task", gcTask)
assert.Equal(t, rootN, gcTaskN.parentTask)
assert.Nil(t, gcTaskN.parentSpan)
// it is allowed that a child task outlives the _span_ in which it was created
// (albeit not its parent task)
e3()
e2()
gcTaskSpan, e5 := WithSpan(gcTask, "granschild-task-span")
gcTaskSpanN := logAndGetTraceNode(t, "granschild-task-span", gcTaskSpan)
assert.Equal(t, gcTaskN, gcTaskSpanN.parentTask)
assert.Equal(t, gcTaskN, gcTaskSpanN.parentSpan)
e5()
e4()
e1()
}
func TestOrphanTasksBecomeNewRootWhenAllAncestorsAreDead(t *testing.T) {
parent, e1 := WithTask(context.Background(), "parent")
_ = logAndGetTraceNode(t, "parent-task", parent)
e1()
var child context.Context
var e2 DoneFunc
require.NotPanics(t, func() {
child, e2 = WithTask(parent, "child")
})
childN := logAndGetTraceNode(t, "child-task", child)
assert.Nil(t, childN.parentTask)
grandchild, _ := WithTask(child, "grandchild")
grandchildN := logAndGetTraceNode(t, "grandchild-task", grandchild)
assert.Equal(t, childN, grandchildN.parentTask)
require.Panics(t, func() { e2() }, "if the parent was alive at creation of child task, wrong termination order remains a panicable offense though")
}
func TestOrphanTaskBecomesChildToNearestLiveAncestor(t *testing.T) {
parent, e1 := WithTask(context.Background(), "parent")
parentN := logAndGetTraceNode(t, "parent-task", parent)
child, e2 := WithTask(parent, "child")
childN := logAndGetTraceNode(t, "child-task", child)
assert.Equal(t, parentN, childN.parentTask)
e2()
grandchild, e3 := WithTask(child, "grandchild")
grandchildN := logAndGetTraceNode(t, "grandchild-task", grandchild)
assert.Equal(t, parentN, grandchildN.parentTask) // child is already dead
require.NotPanics(t, func() {
e3()
e1()
})
}
daemon/logging/trace/trace_unique_concurrent_task_namer.go
+53
View File
@@ -0,0 +1,53 @@
package trace
import (
"fmt"
"strings"
"sync"
"github.com/willf/bitset"
)
type uniqueConcurrentTaskNamer struct {
mtx sync.Mutex
active map[string]*bitset.BitSet
}
// bitvecLengthGauge may be nil
func newUniqueTaskNamer() *uniqueConcurrentTaskNamer {
return &uniqueConcurrentTaskNamer{
active: make(map[string]*bitset.BitSet),
}
}
// appends `#%d` to `name` such that until `done` is called,
// it is guaranteed that `#%d` is not returned a second time for the same `name`
func (namer *uniqueConcurrentTaskNamer) UniqueConcurrentTaskName(name string) (uniqueName string, done func()) {
if strings.Contains(name, "#") {
panic(name)
}
namer.mtx.Lock()
act, ok := namer.active[name]
if !ok {
act = bitset.New(64) // FIXME magic const
namer.active[name] = act
}
id, ok := act.NextClear(0)
if !ok {
// if !ok, all bits are 1 and act.Len() returns the next bit
id = act.Len()
// FIXME unbounded growth without reclamation
}
act.Set(id)
namer.mtx.Unlock()
return fmt.Sprintf("%s#%d", name, id), func() {
namer.mtx.Lock()
defer namer.mtx.Unlock()
act, ok := namer.active[name]
if !ok {
panic("must be initialized upon entry")
}
act.Clear(id)
}
}
daemon/logging/trace/trace_unique_concurrent_task_namer_test.go
+58
View File
@@ -0,0 +1,58 @@
package trace
import (
"fmt"
"sync"
"sync/atomic"
"testing"
"github.com/stretchr/testify/require"
"github.com/willf/bitset"
)
func TestBitsetFeaturesForUniqueConcurrentTaskNamer(t *testing.T) {
var b bitset.BitSet
require.Equal(t, uint(0), b.Len())
require.Equal(t, uint(0), b.Count())
b.Set(0)
require.Equal(t, uint(1), b.Len())
require.Equal(t, uint(1), b.Count())
b.Set(8)
require.Equal(t, uint(9), b.Len())
require.Equal(t, uint(2), b.Count())
b.Set(1)
require.Equal(t, uint(9), b.Len())
require.Equal(t, uint(3), b.Count())
}
func TestUniqueConcurrentTaskNamer(t *testing.T) {
namer := newUniqueTaskNamer()
var wg sync.WaitGroup
const N = 8128
const Q = 23
var fails uint32
var m sync.Map
wg.Add(N)
for i := 0; i < N; i++ {
go func(i int) {
defer wg.Done()
name := fmt.Sprintf("%d", i/Q)
uniqueName, done := namer.UniqueConcurrentTaskName(name)
act, _ := m.LoadOrStore(uniqueName, i)
if act.(int) != i {
atomic.AddUint32(&fails, 1)
}
m.Delete(uniqueName)
done()
}(i)
}
wg.Wait()
require.Equal(t, uint32(0), fails)
}
daemon/main.go
+4 -2
View File
@@ -1,6 +1,8 @@
package daemon
import (
"context"
"github.com/zrepl/zrepl/cli"
"github.com/zrepl/zrepl/logger"
)
@@ -10,7 +12,7 @@ type Logger = logger.Logger
var DaemonCmd = &cli.Subcommand{
Use: "daemon",
Short: "run the zrepl daemon",
Run: func(subcommand *cli.Subcommand, args []string) error {
return Run(subcommand.Config())
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
return Run(ctx, subcommand.Config())
},
}
daemon/pprof.go
+4
View File
@@ -9,7 +9,10 @@ import (
"net/http/pprof"
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/net/websocket"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/logging/trace"
)
type pprofServer struct {
@@ -66,6 +69,7 @@ outer:
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
mux.Handle("/metrics", promhttp.Handler())
mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
go func() {
err := http.Serve(s.listener, mux)
if ctx.Err() != nil {
daemon/prometheus.go
+8 -4
View File
@@ -10,6 +10,7 @@ import (
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
@@ -86,16 +87,19 @@ func (j *prometheusJob) Run(ctx context.Context) {
}
type prometheusJobOutlet struct {
jobName string
}
var _ logger.Outlet = prometheusJobOutlet{}
func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
return prometheusJobOutlet{jobName}
func newPrometheusLogOutlet() prometheusJobOutlet {
return prometheusJobOutlet{}
}
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
jobFieldVal, ok := entry.Fields[logging.JobField].(string)
if !ok {
jobFieldVal = "_nojobid"
}
prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
return nil
}
daemon/pruner/pruner.go
+9 -12
View File
@@ -12,6 +12,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/pruning"
"github.com/zrepl/zrepl/replication/logic/pdu"
@@ -35,17 +36,13 @@ type Logger = logger.Logger
type contextKey int
const contextKeyLogger contextKey = 0
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLogger, log)
}
const (
contextKeyPruneSide contextKey = 1 + iota
)
func GetLogger(ctx context.Context) Logger {
if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
return l
}
return logger.NewNullLogger()
pruneSide := ctx.Value(contextKeyPruneSide).(string)
return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
}
type args struct {
@@ -138,7 +135,7 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
p := &Pruner{
args: args{
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
context.WithValue(ctx, contextKeyPruneSide, "sender"),
target,
receiver,
f.senderRules,
@@ -154,7 +151,7 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, re
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
p := &Pruner{
args: args{
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
context.WithValue(ctx, contextKeyPruneSide, "receiver"),
target,
receiver,
f.receiverRules,
@@ -170,7 +167,7 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
p := &Pruner{
args: args{
ctx,
context.WithValue(ctx, contextKeyPruneSide, "local"),
target,
receiver,
f.keepRules,
daemon/snapper/snapper.go
+30 -44
View File
@@ -8,10 +8,12 @@ import (
"time"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/zfs"
@@ -45,7 +47,6 @@ type snapProgress struct {
type args struct {
ctx context.Context
log Logger
prefix string
interval time.Duration
fsf *filters.DatasetMapFilter
@@ -102,23 +103,10 @@ func (s State) sf() state {
type updater func(u func(*Snapper)) State
type state func(a args, u updater) state
type contextKey int
const (
contextKeyLog contextKey = 0
)
type Logger = logger.Logger
func WithLogger(ctx context.Context, log Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, log)
}
func getLogger(ctx context.Context) Logger {
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
return log
}
return logger.NewNullLogger()
return logging.GetLogger(ctx, logging.SubsysSnapshot)
}
func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
@@ -146,13 +134,12 @@ func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *con
}
func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
getLogger(ctx).Debug("start")
defer getLogger(ctx).Debug("stop")
s.args.snapshotsTaken = snapshotsTaken
s.args.ctx = ctx
s.args.log = getLogger(ctx)
s.args.dryRun = false // for future expansion
u := func(u func(*Snapper)) State {
@@ -190,7 +177,7 @@ func onErr(err error, u updater) state {
case Snapshotting:
s.state = ErrorWait
}
s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
}).sf()
}
@@ -209,7 +196,7 @@ func syncUp(a args, u updater) state {
if err != nil {
return onErr(err, u)
}
syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
syncPoint, err := findSyncPoint(a.ctx, fss, a.prefix, a.interval)
if err != nil {
return onErr(err, u)
}
@@ -266,18 +253,18 @@ func snapshot(a args, u updater) state {
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
l := a.log.
WithField("fs", fs.ToString()).
WithField("snap", snapname)
ctx := logging.WithInjectedField(a.ctx, "fs", fs.ToString())
ctx = logging.WithInjectedField(ctx, "snap", snapname)
hookEnvExtra := hooks.Env{
hooks.EnvFS: fs.ToString(),
hooks.EnvSnapshot: snapname,
}
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
l := getLogger(ctx)
l.Debug("create snapshot")
err = zfs.ZFSSnapshot(a.ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
if err != nil {
l.WithError(err).Error("cannot create snapshot")
}
@@ -290,7 +277,7 @@ func snapshot(a args, u updater) state {
{
filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
if err != nil {
l.WithError(err).Error("unexpected filter error")
getLogger(ctx).WithError(err).Error("unexpected filter error")
fsHadErr = true
goto updateFSState
}
@@ -303,7 +290,7 @@ func snapshot(a args, u updater) state {
plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
if planErr != nil {
fsHadErr = true
l.WithError(planErr).Error("cannot create job hook plan")
getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
goto updateFSState
}
}
@@ -314,15 +301,14 @@ func snapshot(a args, u updater) state {
progress.state = SnapStarted
})
{
l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
l.WithField("report", plan.Report().String()).Debug("begin run job plan")
plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
getLogger(ctx).WithField("report", plan.Report().String()).Debug("begin run job plan")
plan.Run(ctx, a.dryRun)
planReport = plan.Report()
fsHadErr = planReport.HadError() // not just fatal errors
if fsHadErr {
l.WithField("report", planReport.String()).Error("end run job plan with error")
getLogger(ctx).WithField("report", planReport.String()).Error("end run job plan with error")
} else {
l.WithField("report", planReport.String()).Info("end run job plan successful")
getLogger(ctx).WithField("report", planReport.String()).Info("end run job plan successful")
}
}
@@ -342,7 +328,7 @@ func snapshot(a args, u updater) state {
case a.snapshotsTaken <- struct{}{}:
default:
if a.snapshotsTaken != nil {
a.log.Warn("callback channel is full, discarding snapshot update event")
getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
}
}
@@ -355,7 +341,7 @@ func snapshot(a args, u updater) state {
break
}
}
a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
getLogger(a.ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
}
}
@@ -376,7 +362,7 @@ func wait(a args, u updater) state {
lastTick := snapper.lastInvocation
snapper.sleepUntil = lastTick.Add(a.interval)
sleepUntil = snapper.sleepUntil
log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
logFunc := log.Debug
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
logFunc = log.Error
@@ -404,7 +390,7 @@ func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.Dat
var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
// see docs/snapshotting.rst
func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
const (
prioHasVersions int = iota
@@ -426,10 +412,10 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
now := time.Now()
log.Debug("examine filesystem state to find sync point")
getLogger(ctx).Debug("examine filesystem state to find sync point")
for _, d := range fss {
l := log.WithField("fs", d.ToString())
syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(l, now, interval, prefix, d)
ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
if err == findSyncPointFSNoFilesystemVersionsErr {
snaptimes = append(snaptimes, snapTime{
ds: d,
@@ -438,9 +424,9 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
})
} else if err != nil {
hardErrs++
l.WithError(err).Error("cannot determine optimal sync point for this filesystem")
getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
} else {
l.WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
snaptimes = append(snaptimes, snapTime{
ds: d,
prio: prioHasVersions,
@@ -467,7 +453,7 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
})
winnerSyncPoint := snaptimes[0].time
l := log.WithField("syncPoint", winnerSyncPoint.String())
l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
l.Info("determined sync point")
if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
for _, st := range snaptimes {
@@ -483,9 +469,9 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
fsvs, err := zfs.ZFSListFilesystemVersions(context.TODO(), d, zfs.ListFilesystemVersionsOptions{
fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
Types: zfs.Snapshots,
ShortnamePrefix: prefix,
})
@@ -502,7 +488,7 @@ func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval ti
})
latest := fsvs[len(fsvs)-1]
l.WithField("creation", latest.Creation).Debug("found latest snapshot")
getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
since := now.Sub(latest.Creation)
if since < 0 {