[WIP] add and use tracing API as part of package logging

- make `logging.GetLogger(ctx, Subsys)` the authoritative `logger.Logger` factory function
    - the context carries a linked list of injected fields which
      `logging.GetLogger` adds to the logger it returns
- introduce the concept of tasks and spans, also tracked as linked list within ctx
    - [ ] TODO automatic logging of span begins and ends, with a unique
      ID stack that makes it easy to follow a series of log entries in
      concurrent code
    - ability to produce a chrome://tracing-compatible trace file,
      either via an env variable or a `zrepl pprof` subcommand
        - this is not a CPU profile, we already have go pprof for that
        - but it is very useful to visually inspect where the
          replication / snapshotter / pruner spends its time
          ( fixes #307 )

rpc/dataconn/dataconn_client.go

@@ -143,7 +143,6 @@ func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, i
 }
 
 func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
-
 	defer c.log.Debug("ReqRecv returns")
 	conn, err := c.getWire(ctx)
 	if err != nil {

rpc/dataconn/dataconn_server.go

@@ -33,16 +33,33 @@ type Handler interface {
 
 type Logger = logger.Logger
 
+type ContextInterceptorData interface {
+	FullMethod() string
+	ClientIdentity() string
+}
+
+type ContextInterceptor = func(ctx context.Context, data ContextInterceptorData, handler func(ctx context.Context))
+
 type Server struct {
 	h   Handler
 	wi  WireInterceptor
+	ci  ContextInterceptor
 	log Logger
 }
 
-func NewServer(wi WireInterceptor, logger Logger, handler Handler) *Server {
+var noopContextInteceptor = func(ctx context.Context, _ ContextInterceptorData, handler func(context.Context)) {
+	handler(ctx)
+}
+
+// wi and ci may be nil
+func NewServer(wi WireInterceptor, ci ContextInterceptor, logger Logger, handler Handler) *Server {
+	if ci == nil {
+		ci = noopContextInteceptor
+	}
 	return &Server{
 		h:   handler,
 		wi:  wi,
+		ci:  ci,
 		log: logger,
 	}
 }
@@ -93,6 +110,14 @@ func (s *Server) Serve(ctx context.Context, l transport.AuthenticatedListener) {
 	}
 }
 
+type contextInterceptorData struct {
+	fullMethod     string
+	clientIdentity string
+}
+
+func (d contextInterceptorData) FullMethod() string     { return d.fullMethod }
+func (d contextInterceptorData) ClientIdentity() string { return d.clientIdentity }
+
 func (s *Server) serveConn(nc *transport.AuthConn) {
 	s.log.Debug("serveConn begin")
 	defer s.log.Debug("serveConn done")
@@ -117,6 +142,17 @@ func (s *Server) serveConn(nc *transport.AuthConn) {
 	}
 	endpoint := string(header)
 
+	data := contextInterceptorData{
+		fullMethod:     endpoint,
+		clientIdentity: nc.ClientIdentity(),
+	}
+	s.ci(ctx, data, func(ctx context.Context) {
+		s.serveConnRequest(ctx, endpoint, c)
+	})
+}
+
+func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *stream.Conn) {
+
 	reqStructured, err := c.ReadStreamedMessage(ctx, RequestStructuredMaxSize, ReqStructured)
 	if err != nil {
 		s.log.WithError(err).Error("error reading structured part")

rpc/dataconn/microbenchmark/microbenchmark.go

@@ -112,7 +112,7 @@ func server() {
 	orDie(err)
 	l := tcpListener{nl.(*net.TCPListener), "fakeclientidentity"}
 
-	srv := dataconn.NewServer(nil, logger.NewStderrDebugLogger(), devNullHandler{})
+	srv := dataconn.NewServer(nil, nil, logger.NewStderrDebugLogger(), devNullHandler{})
 
 	ctx := context.Background()
 

rpc/dataconn/stream/stream.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"unicode/utf8"
 
@@ -80,9 +81,13 @@ func doWriteStream(ctx context.Context, c *heartbeatconn.Conn, stream io.Reader,
 		err error
 	}
 
+	var wg sync.WaitGroup
+	defer wg.Wait()
 	reads := make(chan read, 5)
 	var stopReading uint32
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		defer close(reads)
 		for atomic.LoadUint32(&stopReading) == 0 {
 			buffer := bufpool.Get(1 << FramePayloadShift)