[WIP] factor out trace functionality into separate package and add Go docs
This commit is contained in:
Christian Schwarz
@@ -0,0 +1,311 @@
|
||||
// package trace provides activity tracing via ctx through Tasks and Spans
|
||||
//
|
||||
// Basic Concepts
|
||||
//
|
||||
// Tracing can be used to identify where a piece of code spends its time.
|
||||
//
|
||||
// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
|
||||
// to understand what happens inside the Go runtime.
|
||||
// However, it is not ideal for application level tracing, in particular if those traces should be understandable
|
||||
// to tech-savvy users (albeit not developers).
|
||||
//
|
||||
// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
|
||||
//
|
||||
// - Neither task nor span is really tangible but instead contained within the context.Context tree
|
||||
// - Tasks represent concurrent activity (i.e. goroutines).
|
||||
// - Spans represent a semantic stack trace within a task.
|
||||
//
|
||||
// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
|
||||
//
|
||||
// go func(ctx context.Context) {
|
||||
// ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
|
||||
// defer endTask()
|
||||
// // ...
|
||||
// }(ctx)
|
||||
//
|
||||
// Within the task, you can open up a hierarchy of spans.
|
||||
// In contrast to tasks, which have can multiple concurrently running child tasks,
|
||||
// spans must nest and not cross the goroutine boundary.
|
||||
//
|
||||
// ctx, endSpan = WithSpan(ctx, "copy-dir")
|
||||
// defer endSpan()
|
||||
// for _, f := range dir.Files() {
|
||||
// func() {
|
||||
// ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
|
||||
// defer endspan()
|
||||
// b, _ := ioutil.ReadFile(f)
|
||||
// _ = ioutil.WriteFile(f + ".copy", b, 0600)
|
||||
// }()
|
||||
// }
|
||||
//
|
||||
// In combination:
|
||||
// ctx, endTask = WithTask(ctx, "copy-dirs")
|
||||
// defer endTask()
|
||||
// for i := range dirs {
|
||||
// go func(dir string) {
|
||||
// ctx, endTask := WithTask(ctx, "copy-dir")
|
||||
// defer endTask()
|
||||
// for _, f := range filesIn(dir) {
|
||||
// func() {
|
||||
// ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
|
||||
// defer endspan()
|
||||
// b, _ := ioutil.ReadFile(f)
|
||||
// _ = ioutil.WriteFile(f + ".copy", b, 0600)
|
||||
// }()
|
||||
// }
|
||||
// }()
|
||||
// }
|
||||
//
|
||||
// Note that a span ends at the time you call endSpan - not before and not after that.
|
||||
// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
|
||||
// the out-of-order endSpan() will panic.
|
||||
//
|
||||
// A similar rule applies to the endTask closure returned by WithTask:
|
||||
// If a task has live child tasks at the time you call endTask(), the call will panic.
|
||||
//
|
||||
// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
|
||||
//
|
||||
//
|
||||
// Best Practices For Naming Tasks And Spans
|
||||
//
|
||||
// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
|
||||
// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
|
||||
// Also, the package appends `#NUM` for each concurrently running instance of a task name.
|
||||
// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
|
||||
// infinite number of horizontal bars in the visualization.
|
||||
//
|
||||
//
|
||||
// Chrome-compatible Tracefile Support
|
||||
//
|
||||
// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
|
||||
// that can be loaded into chrome://tracing .
|
||||
// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
|
||||
//
|
||||
// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
|
||||
// From process start onward, a trace is written to that path.
|
||||
//
|
||||
// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
|
||||
//
|
||||
// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
|
||||
// will not receive further trace output.
|
||||
package trace
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var metrics struct {
|
||||
activeTasks prometheus.Gauge
|
||||
uniqueConcurrentTaskNameBitvecLength *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
func init() {
|
||||
metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "trace",
|
||||
Name: "active_tasks",
|
||||
Help: "number of active (tracing-level) tasks in the daemon",
|
||||
})
|
||||
metrics.uniqueConcurrentTaskNameBitvecLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "trace",
|
||||
Name: "unique_concurrent_task_name_bitvec_length",
|
||||
Help: "length of the bitvec used to find unique names for concurrent tasks",
|
||||
}, []string{"task_name"})
|
||||
}
|
||||
|
||||
func RegisterMetrics(r prometheus.Registerer) {
|
||||
r.MustRegister(metrics.activeTasks)
|
||||
r.MustRegister(metrics.uniqueConcurrentTaskNameBitvecLength)
|
||||
}
|
||||
|
||||
var taskNamer = newUniqueTaskNamer(metrics.uniqueConcurrentTaskNameBitvecLength)
|
||||
|
||||
type traceNode struct {
|
||||
id string
|
||||
annotation string
|
||||
parentTask *traceNode
|
||||
activeChildTasks int32 // only for task nodes, insignificant for span nodes
|
||||
parentSpan *traceNode
|
||||
hasActiveChildSpan int32
|
||||
|
||||
startedAt time.Time
|
||||
endedAt time.Time
|
||||
ended int32
|
||||
}
|
||||
|
||||
// Returned from WithTask or WithSpan.
|
||||
// Must be called once the task or span ends.
|
||||
// See package-level docs for nesting rules.
|
||||
// Wrong call order / forgetting to call it will result in panics.
|
||||
type DoneFunc func()
|
||||
|
||||
// Start a new root task or create a child task of an existing task.
|
||||
//
|
||||
// This is required when starting a new goroutine and
|
||||
// passing an existing task context to it.
|
||||
//
|
||||
// taskName should be a constantand must not contain '#'
|
||||
//
|
||||
// The implementation ensures that,
|
||||
// if multiple tasks with the same name exist simultaneously,
|
||||
// a unique suffix is appended to uniquely identify the task opened with this function.
|
||||
func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
|
||||
|
||||
var parentTask *traceNode
|
||||
nodeI := ctx.Value(contextKeyTraceNode)
|
||||
if nodeI != nil {
|
||||
node := nodeI.(*traceNode)
|
||||
if node.parentSpan != nil {
|
||||
parentTask = node.parentSpan // FIXME review this
|
||||
} else {
|
||||
parentTask = node
|
||||
}
|
||||
}
|
||||
|
||||
taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
|
||||
|
||||
this := &traceNode{
|
||||
id: genID(),
|
||||
annotation: taskName,
|
||||
parentTask: parentTask,
|
||||
activeChildTasks: 0,
|
||||
hasActiveChildSpan: 0,
|
||||
parentSpan: nil,
|
||||
|
||||
startedAt: time.Now(),
|
||||
ended: 0,
|
||||
endedAt: time.Time{},
|
||||
}
|
||||
|
||||
if this.parentTask != nil {
|
||||
atomic.AddInt32(&this.parentTask.activeChildTasks, 1)
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, contextKeyTraceNode, this)
|
||||
|
||||
chrometraceBeginTask(this)
|
||||
|
||||
metrics.activeTasks.Inc()
|
||||
|
||||
endTaskFunc := func() {
|
||||
if nc := atomic.LoadInt32(&this.activeChildTasks); nc != 0 {
|
||||
panic(fmt.Sprintf("this task must have 0 active child tasks, got %v", nc))
|
||||
}
|
||||
|
||||
if !atomic.CompareAndSwapInt32(&this.ended, 0, 1) {
|
||||
return
|
||||
}
|
||||
this.endedAt = time.Now()
|
||||
|
||||
if this.parentTask != nil {
|
||||
if atomic.AddInt32(&this.parentTask.activeChildTasks, -1) < 0 {
|
||||
panic("parent task with negative activeChildTasks count")
|
||||
}
|
||||
}
|
||||
|
||||
chrometraceEndTask(this)
|
||||
|
||||
metrics.activeTasks.Dec()
|
||||
|
||||
taskNameDone()
|
||||
}
|
||||
|
||||
return ctx, endTaskFunc
|
||||
}
|
||||
|
||||
// Start a new span.
|
||||
// Important: ctx must have an active task (see WithTask)
|
||||
func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
|
||||
var parentSpan, parentTask *traceNode
|
||||
nodeI := ctx.Value(contextKeyTraceNode)
|
||||
if nodeI != nil {
|
||||
parentSpan = nodeI.(*traceNode)
|
||||
if parentSpan.parentSpan == nil {
|
||||
parentTask = parentSpan
|
||||
} else {
|
||||
parentTask = parentSpan.parentTask
|
||||
}
|
||||
} else {
|
||||
panic("must be called from within a task")
|
||||
}
|
||||
|
||||
this := &traceNode{
|
||||
id: genID(),
|
||||
annotation: annotation,
|
||||
parentTask: parentTask,
|
||||
parentSpan: parentSpan,
|
||||
hasActiveChildSpan: 0,
|
||||
|
||||
startedAt: time.Now(),
|
||||
ended: 0,
|
||||
endedAt: time.Time{},
|
||||
}
|
||||
|
||||
if !atomic.CompareAndSwapInt32(&parentSpan.hasActiveChildSpan, 0, 1) {
|
||||
panic("already has active child span")
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, contextKeyTraceNode, this)
|
||||
chrometraceBeginSpan(this)
|
||||
|
||||
endTaskFunc := func() {
|
||||
if !atomic.CompareAndSwapInt32(&parentSpan.hasActiveChildSpan, 1, 0) {
|
||||
panic("impl error: hasActiveChildSpan should not change to 0 while we hold it")
|
||||
}
|
||||
this.endedAt = time.Now()
|
||||
|
||||
chrometraceEndSpan(this)
|
||||
}
|
||||
|
||||
return ctx, endTaskFunc
|
||||
}
|
||||
|
||||
func currentTaskNameAndSpanStack(this *traceNode) (taskName string, spanIdStack string) {
|
||||
|
||||
task := this.parentTask
|
||||
if this.parentSpan == nil {
|
||||
task = this
|
||||
}
|
||||
|
||||
var spansInTask []*traceNode
|
||||
for s := this; s != nil; s = s.parentSpan {
|
||||
spansInTask = append(spansInTask, s)
|
||||
}
|
||||
|
||||
var tasks []*traceNode
|
||||
for t := task; t != nil; t = t.parentTask {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
|
||||
var taskIdsRev []string
|
||||
for i := len(tasks) - 1; i >= 0; i-- {
|
||||
taskIdsRev = append(taskIdsRev, tasks[i].id)
|
||||
}
|
||||
|
||||
var spanIdsRev []string
|
||||
for i := len(spansInTask) - 1; i >= 0; i-- {
|
||||
spanIdsRev = append(spanIdsRev, spansInTask[i].id)
|
||||
}
|
||||
|
||||
taskStack := strings.Join(taskIdsRev, "$")
|
||||
spanIdStack = fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
|
||||
|
||||
return task.annotation, spanIdStack
|
||||
}
|
||||
|
||||
func GetSpanStackOrDefault(ctx context.Context, def string) string {
|
||||
if nI := ctx.Value(contextKeyTraceNode); nI != nil {
|
||||
n := nI.(*traceNode)
|
||||
_, spanStack := currentTaskNameAndSpanStack(n)
|
||||
return spanStack
|
||||
} else {
|
||||
return def
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package trace
|
||||
|
||||
// The functions in this file are concerned with the generation
|
||||
// of trace files based on the information from WithTask and WithSpan.
|
||||
//
|
||||
// The emitted trace files are open-ended array of JSON objects
|
||||
// that follow the Chrome trace file format:
|
||||
// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
|
||||
//
|
||||
// The emitted JSON can be loaded into Chrome's chrome://tracing view.
|
||||
//
|
||||
// The trace file can be written to a file whose path is specified in an env file,
|
||||
// and be written to web sockets established on ChrometraceHttpHandler
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/net/websocket"
|
||||
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
var chrometracePID string
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
chrometracePID, err = os.Hostname()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
chrometracePID = fmt.Sprintf("%q", chrometracePID)
|
||||
}
|
||||
|
||||
type chrometraceEvent struct {
|
||||
Cat string `json:"cat,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Stack []string `json:"stack,omitempty"`
|
||||
Phase string `json:"ph"`
|
||||
TimestampUnixMicroseconds int64 `json:"ts"`
|
||||
DurationMicroseconds int64 `json:"dur,omitempty"`
|
||||
Pid string `json:"pid"`
|
||||
Tid string `json:"tid"`
|
||||
Id string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
func chrometraceBeginSpan(s *traceNode) {
|
||||
taskName, _ := currentTaskNameAndSpanStack(s)
|
||||
chrometraceWrite(chrometraceEvent{
|
||||
Name: s.annotation,
|
||||
Phase: "B",
|
||||
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
|
||||
Pid: chrometracePID,
|
||||
Tid: taskName,
|
||||
})
|
||||
}
|
||||
|
||||
func chrometraceEndSpan(s *traceNode) {
|
||||
taskName, _ := currentTaskNameAndSpanStack(s)
|
||||
chrometraceWrite(chrometraceEvent{
|
||||
Name: s.annotation,
|
||||
Phase: "E",
|
||||
TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
|
||||
Pid: chrometracePID,
|
||||
Tid: taskName,
|
||||
})
|
||||
}
|
||||
|
||||
var chrometraceFlowId uint64
|
||||
|
||||
func chrometraceBeginTask(s *traceNode) {
|
||||
chrometraceBeginSpan(s)
|
||||
|
||||
if s.parentTask == nil {
|
||||
return
|
||||
}
|
||||
// beginning of a task that has a parent
|
||||
// => use flow events to link parent and child
|
||||
|
||||
flowId := atomic.AddUint64(&chrometraceFlowId, 1)
|
||||
flowIdStr := fmt.Sprintf("%x", flowId)
|
||||
|
||||
parentTask, _ := currentTaskNameAndSpanStack(s.parentTask)
|
||||
chrometraceWrite(chrometraceEvent{
|
||||
Cat: "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
|
||||
Name: "child-task",
|
||||
Phase: "s",
|
||||
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
|
||||
Pid: chrometracePID,
|
||||
Tid: parentTask,
|
||||
Id: flowIdStr,
|
||||
})
|
||||
|
||||
childTask, _ := currentTaskNameAndSpanStack(s)
|
||||
if parentTask == childTask {
|
||||
panic(parentTask)
|
||||
}
|
||||
chrometraceWrite(chrometraceEvent{
|
||||
Cat: "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
|
||||
Name: "child-task",
|
||||
Phase: "f",
|
||||
TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
|
||||
Pid: chrometracePID,
|
||||
Tid: childTask,
|
||||
Id: flowIdStr,
|
||||
})
|
||||
}
|
||||
|
||||
func chrometraceEndTask(s *traceNode) {
|
||||
chrometraceEndSpan(s)
|
||||
}
|
||||
|
||||
type chrometraceConsumerRegistration struct {
|
||||
w io.Writer
|
||||
// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
|
||||
errored chan error
|
||||
}
|
||||
|
||||
var chrometraceConsumers struct {
|
||||
register chan chrometraceConsumerRegistration
|
||||
consumers map[chrometraceConsumerRegistration]bool
|
||||
write chan []byte
|
||||
}
|
||||
|
||||
func init() {
|
||||
chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
|
||||
chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
|
||||
chrometraceConsumers.write = make(chan []byte)
|
||||
go func() {
|
||||
kickConsumer := func(c chrometraceConsumerRegistration, err error) {
|
||||
debug("chrometrace kicking consumer %#v after error %v", c, err)
|
||||
select {
|
||||
case c.errored <- err:
|
||||
default:
|
||||
}
|
||||
close(c.errored)
|
||||
delete(chrometraceConsumers.consumers, c)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case reg := <-chrometraceConsumers.register:
|
||||
debug("registered chrometrace consumer %#v", reg)
|
||||
chrometraceConsumers.consumers[reg] = true
|
||||
n, err := reg.w.Write([]byte("[\n"))
|
||||
if err != nil {
|
||||
kickConsumer(reg, err)
|
||||
} else if n != 2 {
|
||||
kickConsumer(reg, fmt.Errorf("short write: %v", n))
|
||||
}
|
||||
// successfully registered
|
||||
|
||||
case buf := <-chrometraceConsumers.write:
|
||||
debug("chrometrace write request: %s", string(buf))
|
||||
var r bytes.Reader
|
||||
for c := range chrometraceConsumers.consumers {
|
||||
r.Reset(buf)
|
||||
n, err := io.Copy(c.w, &r)
|
||||
debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
|
||||
if err != nil {
|
||||
kickConsumer(c, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func chrometraceWrite(i interface{}) {
|
||||
var buf bytes.Buffer
|
||||
err := json.NewEncoder(&buf).Encode(i)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
buf.WriteString(",")
|
||||
chrometraceConsumers.write <- buf.Bytes()
|
||||
}
|
||||
|
||||
func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
|
||||
defer conn.Close()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
r := bufio.NewReader(conn)
|
||||
_, _, _ = r.ReadLine() // ignore errors
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
errored := make(chan error, 1)
|
||||
chrometraceConsumers.register <- chrometraceConsumerRegistration{
|
||||
w: conn,
|
||||
errored: errored,
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-errored
|
||||
conn.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
|
||||
|
||||
func init() {
|
||||
if chrometraceFileConsumerPath != "" {
|
||||
var err error
|
||||
f, err := os.Create(chrometraceFileConsumerPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
errored := make(chan error, 1)
|
||||
chrometraceConsumers.register <- chrometraceConsumerRegistration{
|
||||
w: f,
|
||||
errored: errored,
|
||||
}
|
||||
go func() {
|
||||
<-errored
|
||||
f.Close()
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package trace
|
||||
|
||||
import "context"
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyTraceNode contextKey = 1 + iota
|
||||
)
|
||||
|
||||
var contextKeys = []contextKey{
|
||||
contextKeyTraceNode,
|
||||
}
|
||||
|
||||
// WithInherit inherits the task hierarchy from inheritFrom into ctx.
|
||||
// The returned context is a child of ctx, but its task and span are those of inheritFrom.
|
||||
//
|
||||
// Note that in most use cases, callers most likely want to call WithTask since it will most likely
|
||||
// be in some sort of connection handler context.
|
||||
func WithInherit(ctx, inheritFrom context.Context) context.Context {
|
||||
for _, k := range contextKeys {
|
||||
if v := inheritFrom.Value(k); v != nil {
|
||||
ctx = context.WithValue(ctx, k, v) // no shadow
|
||||
}
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// use like this:
|
||||
//
|
||||
// defer WithSpanFromStackUpdateCtx(&existingCtx)()
|
||||
//
|
||||
//
|
||||
func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
|
||||
childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
|
||||
*ctx = childSpanCtx
|
||||
return end
|
||||
}
|
||||
|
||||
// derive task name from call stack (caller's name)
|
||||
func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
|
||||
return WithTask(ctx, getMyCallerOrPanic())
|
||||
}
|
||||
|
||||
// derive task name from call stack (caller's name) and update *ctx
|
||||
// to point to be the child task ctx
|
||||
func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
|
||||
child, end := WithTask(*ctx, getMyCallerOrPanic())
|
||||
*ctx = child
|
||||
return end
|
||||
}
|
||||
|
||||
// create a task and a span within it in one call
|
||||
func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
|
||||
ctx, endTask := WithTask(ctx, task)
|
||||
ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
|
||||
return ctx, func() {
|
||||
endSpan()
|
||||
endTask()
|
||||
}
|
||||
}
|
||||
|
||||
// create a span during which several child tasks are spawned using the `add` function
|
||||
func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
|
||||
var wg sync.WaitGroup
|
||||
ctx, endSpan := WithSpan(ctx, taskGroup)
|
||||
add = func(f func(context.Context)) {
|
||||
wg.Add(1)
|
||||
defer wg.Done()
|
||||
ctx, endTask := WithTask(ctx, taskGroup)
|
||||
defer endTask()
|
||||
f(ctx)
|
||||
}
|
||||
waitEnd = func() {
|
||||
wg.Wait()
|
||||
endSpan()
|
||||
}
|
||||
return ctx, add, waitEnd
|
||||
}
|
||||
|
||||
func getMyCallerOrPanic() string {
|
||||
pc, _, _, ok := runtime.Caller(2)
|
||||
if !ok {
|
||||
panic("cannot get caller")
|
||||
}
|
||||
details := runtime.FuncForPC(pc)
|
||||
if ok && details != nil {
|
||||
const prefix = "github.com/zrepl/zrepl"
|
||||
return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetCallerOrPanic(t *testing.T) {
|
||||
withStackFromCtxMock := func() string {
|
||||
return getMyCallerOrPanic()
|
||||
}
|
||||
ret := withStackFromCtxMock()
|
||||
// zrepl prefix is stripped
|
||||
assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
const debugEnabled = false
|
||||
|
||||
func debug(format string, args ...interface{}) {
|
||||
if !debugEnabled {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
var genIdPRNG = rand.New(rand.NewSource(1))
|
||||
|
||||
func init() {
|
||||
genIdPRNG.Seed(time.Now().UnixNano())
|
||||
genIdPRNG.Seed(int64(os.Getpid()))
|
||||
}
|
||||
|
||||
var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
|
||||
|
||||
func init() {
|
||||
if genIdNumBytes < 1 {
|
||||
panic("trace node id byte length must be at least 1")
|
||||
}
|
||||
}
|
||||
|
||||
func genID() string {
|
||||
var out strings.Builder
|
||||
enc := base64.NewEncoder(base64.RawStdEncoding, &out)
|
||||
buf := make([]byte, genIdNumBytes)
|
||||
for i := 0; i < len(buf); {
|
||||
n, err := genIdPRNG.Read(buf[i:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
i += n
|
||||
}
|
||||
n, err := enc.Write(buf[:])
|
||||
if err != nil || n != len(buf) {
|
||||
panic(err)
|
||||
}
|
||||
if err := enc.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/willf/bitset"
|
||||
)
|
||||
|
||||
type uniqueConcurrentTaskNamer struct {
|
||||
mtx sync.Mutex
|
||||
active map[string]*bitset.BitSet
|
||||
|
||||
bitvecLengthGauge *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// bitvecLengthGauge may be nil
|
||||
func newUniqueTaskNamer(bitvecLengthGauge *prometheus.GaugeVec) *uniqueConcurrentTaskNamer {
|
||||
return &uniqueConcurrentTaskNamer{
|
||||
active: make(map[string]*bitset.BitSet),
|
||||
bitvecLengthGauge: bitvecLengthGauge,
|
||||
}
|
||||
}
|
||||
|
||||
// appends `#%d` to `name` such that until `done` is called,
|
||||
// it is guaranteed that `#%d` is not returned a second time for the same `name`
|
||||
func (namer *uniqueConcurrentTaskNamer) UniqueConcurrentTaskName(name string) (uniqueName string, done func()) {
|
||||
if strings.Contains(name, "#") {
|
||||
panic(name)
|
||||
}
|
||||
namer.mtx.Lock()
|
||||
act, ok := namer.active[name]
|
||||
if !ok {
|
||||
act = bitset.New(64) // FIXME magic const
|
||||
namer.active[name] = act
|
||||
}
|
||||
id, ok := act.NextClear(0)
|
||||
if !ok {
|
||||
// if !ok, all bits are 1 and act.Len() returns the next bit
|
||||
id = act.Len()
|
||||
// FIXME unbounded growth without reclamation
|
||||
}
|
||||
act.Set(id)
|
||||
namer.mtx.Unlock()
|
||||
|
||||
if namer.bitvecLengthGauge != nil {
|
||||
namer.bitvecLengthGauge.WithLabelValues(name).Set(float64(act.Len()))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s#%d", name, id), func() {
|
||||
namer.mtx.Lock()
|
||||
defer namer.mtx.Unlock()
|
||||
act, ok := namer.active[name]
|
||||
if !ok {
|
||||
panic("must be initialized upon entry")
|
||||
}
|
||||
act.Clear(id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package trace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/willf/bitset"
|
||||
)
|
||||
|
||||
func TestBitsetFeaturesForUniqueConcurrentTaskNamer(t *testing.T) {
|
||||
var b bitset.BitSet
|
||||
require.Equal(t, uint(0), b.Len())
|
||||
require.Equal(t, uint(0), b.Count())
|
||||
|
||||
b.Set(0)
|
||||
require.Equal(t, uint(1), b.Len())
|
||||
require.Equal(t, uint(1), b.Count())
|
||||
|
||||
b.Set(8)
|
||||
require.Equal(t, uint(9), b.Len())
|
||||
require.Equal(t, uint(2), b.Count())
|
||||
|
||||
b.Set(1)
|
||||
require.Equal(t, uint(9), b.Len())
|
||||
require.Equal(t, uint(3), b.Count())
|
||||
}
|
||||
|
||||
func TestUniqueConcurrentTaskNamer(t *testing.T) {
|
||||
namer := newUniqueTaskNamer(nil)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
const N = 8128
|
||||
const Q = 23
|
||||
|
||||
var fails uint32
|
||||
var m sync.Map
|
||||
wg.Add(N)
|
||||
for i := 0; i < N; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
name := fmt.Sprintf("%d", i/Q)
|
||||
uniqueName, done := namer.UniqueConcurrentTaskName(name)
|
||||
act, _ := m.LoadOrStore(uniqueName, i)
|
||||
if act.(int) != i {
|
||||
atomic.AddUint32(&fails, 1)
|
||||
}
|
||||
m.Delete(uniqueName)
|
||||
done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Equal(t, uint32(0), fails)
|
||||
}
|
||||
Reference in New Issue
Block a user