move implementation to internal/ directory (#828)

internal/util/bandwidthlimit/bandwidthlimit.go

@@ -0,0 +1,79 @@
+package bandwidthlimit
+
+import (
+	"errors"
+	"io"
+
+	"github.com/juju/ratelimit"
+)
+
+type Wrapper interface {
+	WrapReadCloser(io.ReadCloser) io.ReadCloser
+}
+
+type Config struct {
+	// Units in this struct are in _bytes_.
+
+	Max            int64 // < 0 means no limit, BucketCapacity is irrelevant then
+	BucketCapacity int64
+}
+
+func NoLimitConfig() Config {
+	return Config{
+		Max:            -1,
+		BucketCapacity: -1,
+	}
+}
+
+func ValidateConfig(conf Config) error {
+	if conf.BucketCapacity == 0 {
+		return errors.New("BucketCapacity must not be zero")
+	}
+	return nil
+}
+
+func WrapperFromConfig(conf Config) Wrapper {
+	if err := ValidateConfig(conf); err != nil {
+		panic(err)
+	}
+
+	if conf.Max < 0 {
+		return noLimit{}
+	}
+
+	return &withLimit{
+		bucket: ratelimit.NewBucketWithRate(float64(conf.Max), conf.BucketCapacity),
+	}
+}
+
+type noLimit struct{}
+
+func (_ noLimit) WrapReadCloser(rc io.ReadCloser) io.ReadCloser { return rc }
+
+type withLimit struct {
+	bucket *ratelimit.Bucket
+}
+
+func (l *withLimit) WrapReadCloser(rc io.ReadCloser) io.ReadCloser {
+	return WrapReadCloser(rc, l.bucket)
+}
+
+type withLimitReadCloser struct {
+	orig    io.Closer
+	limited io.Reader
+}
+
+func (r *withLimitReadCloser) Read(buf []byte) (int, error) {
+	return r.limited.Read(buf)
+}
+
+func (r *withLimitReadCloser) Close() error {
+	return r.orig.Close()
+}
+
+func WrapReadCloser(rc io.ReadCloser, bucket *ratelimit.Bucket) io.ReadCloser {
+	return &withLimitReadCloser{
+		limited: ratelimit.Reader(rc, bucket),
+		orig:    rc,
+	}
+}

internal/util/bandwidthlimit/bandwidthlimit_test.go

@@ -0,0 +1,19 @@
+package bandwidthlimit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNoLimitConfig(t *testing.T) {
+
+	conf := NoLimitConfig()
+
+	err := ValidateConfig(conf)
+	require.NoError(t, err)
+
+	require.NotPanics(t, func() {
+		_ = WrapperFromConfig(conf)
+	})
+}

internal/util/bytecounter/bytecounter_readcloser.go

@@ -0,0 +1,42 @@
+package bytecounter
+
+import (
+	"io"
+	"sync/atomic"
+)
+
+// ReadCloser wraps an io.ReadCloser, reimplementing
+// its interface and counting the bytes written to during copying.
+type ReadCloser interface {
+	io.ReadCloser
+	Count() uint64
+}
+
+// NewReadCloser wraps rc.
+func NewReadCloser(rc io.ReadCloser) ReadCloser {
+	return &readCloser{rc, 0}
+}
+
+type readCloser struct {
+	rc    io.ReadCloser
+	count uint64
+}
+
+func (r *readCloser) Count() uint64 {
+	return atomic.LoadUint64(&r.count)
+}
+
+var _ io.ReadCloser = &readCloser{}
+
+func (r *readCloser) Close() error {
+	return r.rc.Close()
+}
+
+func (r *readCloser) Read(p []byte) (int, error) {
+	n, err := r.rc.Read(p)
+	if n < 0 {
+		panic("expecting n >= 0")
+	}
+	atomic.AddUint64(&r.count, uint64(n))
+	return n, err
+}

internal/util/chainedio/chainedio_readcloser.go

@@ -0,0 +1,43 @@
+package chainedio
+
+import "io"
+
+type ChainedReadCloser struct {
+	readers   []io.Reader
+	curReader int
+}
+
+func NewChainedReader(reader ...io.Reader) *ChainedReadCloser {
+	return &ChainedReadCloser{
+		readers:   reader,
+		curReader: 0,
+	}
+}
+
+func (c *ChainedReadCloser) Read(buf []byte) (n int, err error) {
+
+	n = 0
+
+	for c.curReader < len(c.readers) {
+		n, err = c.readers[c.curReader].Read(buf)
+		if err == io.EOF {
+			c.curReader++
+			continue
+		}
+		break
+	}
+	if c.curReader == len(c.readers) {
+		err = io.EOF // actually, there was no gap
+	}
+
+	return
+}
+
+func (c *ChainedReadCloser) Close() error {
+	for _, r := range c.readers {
+		if c, ok := r.(io.Closer); ok {
+			c.Close() // TODO debug log error?
+		}
+	}
+	return nil
+}

internal/util/chainlock/chainlock.go

@@ -0,0 +1,46 @@
+// package chainlock implements a mutex whose Lock and Unlock
+// methods return the lock itself, to enable chaining.
+//
+// Intended Usage
+//
+//	  defer s.lock().unlock()
+//	  // drop lock while waiting for wait group
+//	  func() {
+//		     defer a.l.Unlock().Lock()
+//		     fssesDone.Wait()
+//		 }()
+package chainlock
+
+import "sync"
+
+type L struct {
+	mtx sync.Mutex
+}
+
+func New() *L {
+	return &L{}
+}
+
+func (l *L) Lock() *L {
+	l.mtx.Lock()
+	return l
+}
+
+func (l *L) Unlock() *L {
+	l.mtx.Unlock()
+	return l
+}
+
+func (l *L) NewCond() *sync.Cond {
+	return sync.NewCond(&l.mtx)
+}
+
+func (l *L) DropWhile(f func()) {
+	defer l.Unlock().Lock()
+	f()
+}
+
+func (l *L) HoldWhile(f func()) {
+	defer l.Lock().Unlock()
+	f()
+}

internal/util/choices/choices.go

@@ -0,0 +1,98 @@
+// Package choice implements a flag.Value type that accepts a set of choices.
+//
+// See test cases or grep the code base for usage hints.
+package choices
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+)
+
+type Choices struct {
+	choices    map[string]interface{}
+	typeString string
+	value      interface{}
+}
+
+var _ flag.Value = (*Choices)(nil)
+
+func new(pairs ...interface{}) Choices {
+	if (len(pairs) % 2) != 0 {
+		panic("must provide a sequence of key value pairs")
+	}
+	c := Choices{
+		choices: make(map[string]interface{}, len(pairs)/2),
+		value:   nil,
+	}
+	for i := 0; i < len(pairs); {
+		key, ok := pairs[i].(string)
+		if !ok {
+			panic(fmt.Sprintf("argument %d is %T but should be a string, value: %#v", i, pairs[i], pairs[i]))
+		}
+		c.choices[key] = pairs[i+1]
+		i += 2
+	}
+	c.typeString = strings.Join(c.choicesList(true), ",") // overrideable by setter
+	return c
+}
+
+func (c *Choices) Init(pairs ...interface{}) {
+	*c = new(pairs...)
+}
+
+func (c Choices) choicesList(escaped bool) []string {
+	keys := make([]string, len(c.choices))
+	i := 0
+	for k := range c.choices {
+		e := k
+		if escaped {
+			e = fmt.Sprintf("%q", k)
+		}
+		keys[i] = e
+		i += 1
+	}
+	return keys
+}
+
+func (c Choices) Usage() string {
+	return fmt.Sprintf("one of %s", strings.Join(c.choicesList(true), ","))
+}
+
+func (c Choices) InputForChoice(v interface{}) (string, error) {
+	for input, choice := range c.choices {
+		if choice == v {
+			return input, nil
+		}
+	}
+	return "", fmt.Errorf("choice not registered at .Init(): %v", v)
+}
+
+func (c *Choices) SetDefaultValue(v interface{}) {
+	c.value = v
+}
+
+func (c Choices) Value() interface{} {
+	return c.value
+}
+
+func (c *Choices) Set(input string) error {
+	v, ok := c.choices[input]
+	if !ok {
+		return fmt.Errorf("invalid value %q: must be one of %s", input, c.Usage())
+	}
+	c.value = v
+	return nil
+}
+
+func (c *Choices) String() string {
+	return "" // c.value.(fmt.Stringer).String()
+}
+
+func (c *Choices) SetTypeString(ts string) {
+	c.typeString = ts
+}
+
+func (c *Choices) Type() string {
+	return c.typeString
+}

internal/util/choices/choices_test.go

@@ -0,0 +1,50 @@
+package choices_test
+
+import (
+	"bytes"
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/util/choices"
+)
+
+func TestChoices(t *testing.T) {
+
+	var c choices.Choices
+
+	fs := flag.NewFlagSet("testset", flag.ContinueOnError)
+	c.Init("append", os.O_APPEND, "overwrite", os.O_TRUNC|os.O_CREATE)
+	fs.Var(&c, "mode", c.Usage())
+	var o bytes.Buffer
+	fs.SetOutput(&o)
+
+	fs.Usage()
+	usage := o.String()
+	o.Reset()
+
+	t.Logf("usage:\n%s", usage)
+	require.Contains(t, usage, "\"append\"")
+	require.Contains(t, usage, "\"overwrite\"")
+
+	err := fs.Parse([]string{"-mode", "append"})
+	require.NoError(t, err)
+	o.Reset()
+	require.Equal(t, os.O_APPEND, c.Value())
+
+	c.SetDefaultValue(nil)
+	err = fs.Parse([]string{})
+	require.NoError(t, err)
+	o.Reset()
+	require.Nil(t, c.Value())
+
+	// a little whitebox testing: this is allowed ATM, we don't check that the default value was specified as a choice in init
+	c.SetDefaultValue(os.O_RDWR)
+	err = fs.Parse([]string{})
+	require.NoError(t, err)
+	o.Reset()
+	require.Equal(t, os.O_RDWR, c.Value())
+
+}

internal/util/chunking/chunking.go

@@ -0,0 +1,204 @@
+package chunking
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+)
+
+var ChunkBufSize uint32 = 32 * 1024
+var ChunkHeaderByteOrder = binary.LittleEndian
+
+type Unchunker struct {
+	ChunkCount          int
+	in                  io.Reader
+	remainingChunkBytes uint32
+	finishErr           error
+}
+
+func NewUnchunker(conn io.Reader) *Unchunker {
+	return &Unchunker{
+		in:                  conn,
+		remainingChunkBytes: 0,
+	}
+}
+
+func (c *Unchunker) Read(b []byte) (n int, err error) {
+
+	if c.finishErr != nil {
+		return 0, c.finishErr
+	}
+
+	if c.remainingChunkBytes == 0 {
+
+		var nextChunkLen uint32
+		err = binary.Read(c.in, ChunkHeaderByteOrder, &nextChunkLen)
+		if err != nil {
+			c.finishErr = err // can't handle this
+			return
+		}
+
+		// A chunk of len 0 indicates end of stream
+		if nextChunkLen == 0 {
+			c.finishErr = io.EOF
+			return 0, c.finishErr
+		}
+
+		c.remainingChunkBytes = nextChunkLen
+		c.ChunkCount++
+
+	}
+
+	if c.remainingChunkBytes == 0 {
+		panic("internal inconsistency: c.remainingChunkBytes must be > 0")
+	}
+	if len(b) <= 0 {
+		panic("cannot read into buffer of length 0")
+	}
+
+	maxRead := min(int(c.remainingChunkBytes), len(b))
+
+	n, err = c.in.Read(b[0:maxRead])
+	if err != nil {
+		return
+	}
+	c.remainingChunkBytes -= uint32(n)
+
+	return
+
+}
+
+func (c *Unchunker) Close() (err error) {
+
+	buf := make([]byte, 4096)
+	for err == nil {
+		_, err = c.Read(buf)
+		if err == io.EOF {
+			err = nil
+			break
+		}
+	}
+	return
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+type Chunker struct {
+	ChunkCount          int
+	in                  io.Reader
+	inEOF               bool
+	remainingChunkBytes int
+	payloadBufLen       int
+	payloadBuf          []byte
+	headerBuf           *bytes.Buffer
+	finalHeaderBuffered bool
+}
+
+func NewChunker(conn io.Reader) Chunker {
+	return NewChunkerSized(conn, ChunkBufSize)
+}
+
+func NewChunkerSized(conn io.Reader, chunkSize uint32) Chunker {
+
+	buf := make([]byte, int(chunkSize)-binary.Size(chunkSize))
+
+	return Chunker{
+		in:                  conn,
+		remainingChunkBytes: 0,
+		payloadBufLen:       0,
+		payloadBuf:          buf,
+		headerBuf:           &bytes.Buffer{},
+	}
+
+}
+
+func (c *Chunker) Read(b []byte) (n int, err error) {
+
+	if len(b) == 0 {
+		panic("unexpected empty output buffer")
+	}
+
+	if c.inEOF && c.finalHeaderBuffered && c.headerBuf.Len() == 0 { // all bufs empty and no more bytes to expect
+		return 0, io.EOF
+	}
+
+	n = 0
+
+	if c.headerBuf.Len() > 0 { // first drain the header buf
+		nh, err := c.headerBuf.Read(b[n:])
+		if nh > 0 {
+			n += nh
+		}
+		if nh == 0 || (err != nil && err != io.EOF) {
+			panic("unexpected behavior: in-memory buffer should not throw errors")
+		}
+		if c.headerBuf.Len() != 0 {
+			return n, nil // finish writing the header before we can proceed with payload
+		}
+		if c.finalHeaderBuffered {
+			// we just wrote the final header
+			return n, io.EOF
+		}
+	}
+
+	if c.remainingChunkBytes > 0 { // then drain the payload buf
+
+		npl := copy(b[n:], c.payloadBuf[c.payloadBufLen-c.remainingChunkBytes:c.payloadBufLen])
+		c.remainingChunkBytes -= npl
+		if c.remainingChunkBytes < 0 {
+			panic("unexpected behavior, copy() should not copy more than max(cap(), len())")
+		}
+		n += npl
+	}
+
+	if c.remainingChunkBytes == 0 && !c.inEOF { // fillup bufs
+
+		newPayloadLen, err := c.in.Read(c.payloadBuf)
+
+		if newPayloadLen > 0 {
+			c.payloadBufLen = newPayloadLen
+			c.remainingChunkBytes = newPayloadLen
+		}
+		if err == io.EOF {
+			c.inEOF = true
+		} else if err != nil {
+			return n, err
+		}
+		if newPayloadLen == 0 { // apparently, this happens with some Readers
+			c.finalHeaderBuffered = true
+		}
+
+		// Fill header buf
+		{
+			c.headerBuf.Reset()
+			nextChunkLen := uint32(newPayloadLen)
+			err := binary.Write(c.headerBuf, ChunkHeaderByteOrder, nextChunkLen)
+			if err != nil {
+				panic("unexpected error, write to in-memory buffer should not throw error")
+			}
+		}
+
+		if c.headerBuf.Len() == 0 {
+			panic("unexpected empty header buf")
+		}
+
+	} else if c.remainingChunkBytes == 0 && c.inEOF && !c.finalHeaderBuffered {
+
+		c.headerBuf.Reset()
+		err := binary.Write(c.headerBuf, ChunkHeaderByteOrder, uint32(0))
+		if err != nil {
+			panic("unexpected error, write to in-memory buffer should not throw error [2]")
+		}
+		c.finalHeaderBuffered = true
+
+	}
+
+	return
+
+}

internal/util/chunking/chunking_test.go

@@ -0,0 +1,74 @@
+package chunking
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnchunker(t *testing.T) {
+
+	buf := bytes.Buffer{}
+	binary.Write(&buf, ChunkHeaderByteOrder, uint32(2))
+	buf.WriteByte(0xca)
+	buf.WriteByte(0xfe)
+	binary.Write(&buf, ChunkHeaderByteOrder, uint32(0))
+	buf.WriteByte(0xff) // sentinel, should not be read
+
+	un := NewUnchunker(&buf)
+
+	recv := bytes.Buffer{}
+	n, err := io.Copy(&recv, un)
+	assert.Nil(t, err)
+	assert.Equal(t, int64(2), n)
+	assert.Equal(t, []byte{0xca, 0xfe}, recv.Bytes())
+
+}
+
+func TestChunker(t *testing.T) {
+
+	buf := bytes.Buffer{}
+	buf.WriteByte(0xca)
+	buf.WriteByte(0xfe)
+
+	ch := NewChunker(&buf)
+
+	chunked := bytes.Buffer{}
+	n, err := io.Copy(&chunked, &ch)
+	assert.Nil(t, err)
+	assert.Equal(t, int64(4+2+4), n)
+	assert.Equal(t, []byte{0x2, 0x0, 0x0, 0x0, 0xca, 0xfe, 0x0, 0x0, 0x0, 0x0}, chunked.Bytes())
+
+}
+
+func TestUnchunkerUnchunksChunker(t *testing.T) {
+
+	f := func(b []byte) bool {
+
+		buf := bytes.NewBuffer(b)
+		ch := NewChunker(buf)
+		unch := NewUnchunker(&ch)
+		var tx bytes.Buffer
+		_, err := io.Copy(&tx, unch)
+		if err != nil {
+			return false
+		}
+
+		return reflect.DeepEqual(b, tx.Bytes())
+	}
+
+	cfg := quick.Config{
+		MaxCount:      3 * int(ChunkBufSize),
+		MaxCountScale: 2.0,
+	}
+
+	if err := quick.Check(f, &cfg); err != nil {
+		t.Error(err)
+	}
+
+}

internal/util/circlog/circlog.go

@@ -0,0 +1,175 @@
+package circlog
+
+import (
+	"fmt"
+	"math/bits"
+	"sync"
+)
+
+const CIRCULARLOG_INIT_SIZE int = 32 << 10
+
+type CircularLog struct {
+	buf         []byte
+	size        int
+	max         int
+	written     int
+	writeCursor int
+	/*
+		Mutex prevents:
+		concurrent writes:
+			buf, size, written, writeCursor in Write([]byte)
+			buf, writeCursor in Reset()
+		data races vs concurrent Write([]byte) calls:
+			size in Size()
+			size, writeCursor in Len()
+			buf, size, written, writeCursor in Bytes()
+	*/
+	mtx sync.Mutex
+}
+
+func MustNewCircularLog(max int) *CircularLog {
+	log, err := NewCircularLog(max)
+	if err != nil {
+		panic(err)
+	}
+	return log
+}
+
+func NewCircularLog(max int) (*CircularLog, error) {
+	if max <= 0 {
+		return nil, fmt.Errorf("max must be positive")
+	}
+
+	return &CircularLog{
+		size: CIRCULARLOG_INIT_SIZE,
+		buf:  make([]byte, CIRCULARLOG_INIT_SIZE),
+		max:  max,
+	}, nil
+}
+
+func nextPow2Int(n int) int {
+	if n < 1 {
+		panic("can only round up positive integers")
+	}
+	r := uint(n)
+	// Credit: https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
+	r--
+	// Assume at least a 32 bit integer
+	r |= r >> 1
+	r |= r >> 2
+	r |= r >> 4
+	r |= r >> 8
+	r |= r >> 16
+	if bits.UintSize == 64 {
+		r |= r >> 32
+	}
+	r++
+	// Can't exceed max positive int value
+	if r > ^uint(0)>>1 {
+		panic("rounded to larger than int()")
+	}
+	return int(r)
+}
+
+func (cl *CircularLog) Write(data []byte) (int, error) {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+	n := len(data)
+
+	// Keep growing the buffer by doubling until
+	// hitting cl.max
+	bufAvail := cl.size - cl.writeCursor
+	// If cl.writeCursor wrapped on the last write,
+	// then the buffer is full, not empty.
+	if cl.writeCursor == 0 && cl.written > 0 {
+		bufAvail = 0
+	}
+	if n > bufAvail && cl.size < cl.max {
+		// Add to size, not writeCursor, so as
+		// to not resize multiple times if this
+		// Write() immediately fills up the buffer
+		newSize := nextPow2Int(cl.size + n)
+		if newSize > cl.max {
+			newSize = cl.max
+		}
+		newBuf := make([]byte, newSize)
+		// Reset write cursor to old size if wrapped
+		if cl.writeCursor == 0 && cl.written > 0 {
+			cl.writeCursor = cl.size
+		}
+		copy(newBuf, cl.buf[:cl.writeCursor])
+		cl.buf = newBuf
+		cl.size = newSize
+	}
+
+	// If data to be written is larger than the max size,
+	// discard all but the last cl.size bytes
+	if n > cl.size {
+		data = data[n-cl.size:]
+		// Overwrite the beginning of data
+		// with a string indicating truncation
+		copy(data, []byte("(...)"))
+	}
+	// First copy data to the end of buf. If that wasn't
+	// all of data, then copy the rest to the beginning
+	// of buf.
+	copied := copy(cl.buf[cl.writeCursor:], data)
+	if copied < n {
+		copied += copy(cl.buf, data[copied:])
+	}
+
+	cl.writeCursor = ((cl.writeCursor + copied) % cl.size)
+	cl.written += copied
+	return copied, nil
+}
+
+func (cl *CircularLog) Size() int {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+	return cl.size
+}
+
+func (cl *CircularLog) Len() int {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+	if cl.written >= cl.size {
+		return cl.size
+	} else {
+		return cl.writeCursor
+	}
+}
+
+func (cl *CircularLog) TotalWritten() int {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+
+	return cl.written
+}
+
+func (cl *CircularLog) Reset() {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+	cl.writeCursor = 0
+	cl.buf = make([]byte, CIRCULARLOG_INIT_SIZE)
+	cl.size = CIRCULARLOG_INIT_SIZE
+}
+
+func (cl *CircularLog) Bytes() []byte {
+	cl.mtx.Lock()
+	defer cl.mtx.Unlock()
+	switch {
+	case cl.written >= cl.size && cl.writeCursor == 0:
+		return cl.buf
+	case cl.written > cl.size:
+		ret := make([]byte, cl.size)
+		copy(ret, cl.buf[cl.writeCursor:])
+		copy(ret[cl.size-cl.writeCursor:], cl.buf[:cl.writeCursor])
+		return ret
+	default:
+		return cl.buf[:cl.writeCursor]
+	}
+}
+
+func (cl *CircularLog) String() string {
+	return string(cl.Bytes())
+}

internal/util/circlog/circlog_test.go

@@ -0,0 +1,144 @@
+package circlog_test
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/util/circlog"
+)
+
+func TestCircularLog(t *testing.T) {
+	var maxCircularLogSize int = 1 << 20
+
+	writeFixedSize := func(w io.Writer, size int) (int, error) {
+		pattern := []byte{0xDE, 0xAD, 0xBE, 0xEF}
+		writeBytes := bytes.Repeat(pattern, size/4)
+		if len(writeBytes) < size {
+			writeBytes = append(writeBytes, pattern[:size-len(writeBytes)]...)
+		}
+		n, err := w.Write(writeBytes)
+		if err != nil {
+			return n, err
+		}
+		return n, nil
+	}
+
+	t.Run("negative-size-error", func(t *testing.T) {
+		_, err := circlog.NewCircularLog(-1)
+		require.EqualError(t, err, "max must be positive")
+	})
+
+	t.Run("no-resize", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+		writeSize := circlog.CIRCULARLOG_INIT_SIZE - 1
+		_, err = writeFixedSize(log, writeSize)
+		require.NoError(t, err)
+		finalSize := log.Size()
+		require.Equal(t, initSize, finalSize)
+		require.Equal(t, writeSize, log.Len())
+	})
+	t.Run("one-resize", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+		writeSize := circlog.CIRCULARLOG_INIT_SIZE + 1
+		_, err = writeFixedSize(log, writeSize)
+		require.NoError(t, err)
+		finalSize := log.Size()
+		require.Greater(t, finalSize, initSize)
+		require.LessOrEqual(t, finalSize, maxCircularLogSize)
+		require.Equal(t, writeSize, log.Len())
+	})
+	t.Run("reset", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+		_, err = writeFixedSize(log, maxCircularLogSize)
+		require.NoError(t, err)
+		log.Reset()
+		finalSize := log.Size()
+		require.Equal(t, initSize, finalSize)
+	})
+	t.Run("wrap-exactly-maximum", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+		writeSize := maxCircularLogSize / 4
+		for written := 0; written < maxCircularLogSize; {
+			n, err := writeFixedSize(log, writeSize)
+			written += n
+			require.NoError(t, err)
+		}
+		finalSize := log.Size()
+		require.Greater(t, finalSize, initSize)
+		require.Equal(t, finalSize, maxCircularLogSize)
+		require.Equal(t, finalSize, log.Len())
+	})
+	t.Run("wrap-partial", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+
+		startSentinel := []byte{0x00, 0x00, 0x00, 0x00}
+		_, err = log.Write(startSentinel)
+		require.NoError(t, err)
+
+		nWritesToFill := 4
+		writeSize := maxCircularLogSize / nWritesToFill
+		for i := 0; i < (nWritesToFill + 1); i++ {
+			_, err := writeFixedSize(log, writeSize)
+			require.NoError(t, err)
+		}
+
+		endSentinel := []byte{0xFF, 0xFF, 0xFF, 0xFF}
+		_, err = log.Write(endSentinel)
+		require.NoError(t, err)
+
+		finalSize := log.Size()
+		require.Greater(t, finalSize, initSize)
+		require.Greater(t, log.TotalWritten(), finalSize)
+		require.Equal(t, finalSize, maxCircularLogSize)
+		require.Equal(t, finalSize, log.Len())
+
+		logBytes := log.Bytes()
+
+		require.Equal(t, endSentinel, logBytes[len(logBytes)-len(endSentinel):])
+		require.NotEqual(t, startSentinel, logBytes[:len(startSentinel)])
+	})
+	t.Run("overflow-write", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		initSize := log.Size()
+		writeSize := maxCircularLogSize + 1
+		n, err := writeFixedSize(log, writeSize)
+		require.NoError(t, err)
+		finalSize := log.Size()
+		require.Less(t, n, writeSize)
+		require.Greater(t, finalSize, initSize)
+		require.Equal(t, finalSize, maxCircularLogSize)
+		logBytes := log.Bytes()
+		require.Equal(t, []byte("(...)"), logBytes[:5])
+	})
+	t.Run("stringify", func(t *testing.T) {
+		log, err := circlog.NewCircularLog(maxCircularLogSize)
+		require.NoError(t, err)
+
+		writtenString := "A harmful truth is better than a useful lie."
+		n, err := log.Write([]byte(writtenString))
+		require.NoError(t, err)
+		require.Equal(t, len(writtenString), n)
+		loggedString := log.String()
+		require.Equal(t, writtenString, loggedString)
+	})
+}

internal/util/circlog/nextpow2int_test.go

@@ -0,0 +1,16 @@
+package circlog
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNextPow2Int(t *testing.T) {
+	require.Equal(t, 512, nextPow2Int(512), "a power of 2 should round to itself")
+	require.Equal(t, 1024, nextPow2Int(513), "should round up to the next power of 2")
+	require.PanicsWithValue(t, "can only round up positive integers", func() { nextPow2Int(0) }, "unimplemented: zero is not positive; corner case")
+	require.PanicsWithValue(t, "can only round up positive integers", func() { nextPow2Int(-1) }, "unimplemented: cannot round up negative numbers")
+	maxInt := int((^uint(0)) >> 1)
+	require.PanicsWithValue(t, "rounded to larger than int()", func() { nextPow2Int(maxInt - 1) }, "cannot round to a number bigger than the int type")
+}

internal/util/connlogger/connlogger.go

@@ -0,0 +1,67 @@
+package connlogger
+
+import (
+	"net"
+	"os"
+)
+
+type NetConnLogger struct {
+	net.Conn
+	ReadFile  *os.File
+	WriteFile *os.File
+}
+
+func NewNetConnLogger(conn net.Conn, readlog, writelog string) (l *NetConnLogger, err error) {
+	l = &NetConnLogger{
+		Conn: conn,
+	}
+	flags := os.O_CREATE | os.O_WRONLY
+	if readlog != "" {
+		if l.ReadFile, err = os.OpenFile(readlog, flags, 0600); err != nil {
+			return
+		}
+	}
+	if writelog != "" {
+		if l.WriteFile, err = os.OpenFile(writelog, flags, 0600); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (c *NetConnLogger) Read(buf []byte) (n int, err error) {
+	n, err = c.Conn.Read(buf)
+	if c.WriteFile != nil {
+		if _, writeErr := c.ReadFile.Write(buf[0:n]); writeErr != nil {
+			panic(writeErr)
+		}
+	}
+	return
+}
+
+func (c *NetConnLogger) Write(buf []byte) (n int, err error) {
+	n, err = c.Conn.Write(buf)
+	if c.ReadFile != nil {
+		if _, writeErr := c.WriteFile.Write(buf[0:n]); writeErr != nil {
+			panic(writeErr)
+		}
+	}
+	return
+}
+func (c *NetConnLogger) Close() (err error) {
+	err = c.Conn.Close()
+	if err != nil {
+		return
+	}
+	if c.ReadFile != nil {
+		if err := c.ReadFile.Close(); err != nil {
+			panic(err)
+		}
+	}
+	if c.WriteFile != nil {
+		if err := c.WriteFile.Close(); err != nil {
+			panic(err)
+		}
+	}
+	return
+}

internal/util/datasizeunit/datasizeunit.go

@@ -0,0 +1,86 @@
+package datasizeunit
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Bits struct {
+	bits float64
+}
+
+func (b Bits) ToBits() float64    { return b.bits }
+func (b Bits) ToBytes() float64   { return b.bits / 8 }
+func FromBytesInt64(i int64) Bits { return Bits{float64(i) * 8} }
+
+var datarateRegex = regexp.MustCompile(`^([-0-9\.]*)\s*(bit|(|K|Ki|M|Mi|G|Gi|T|Ti)([bB]))$`)
+
+func (r *Bits) UnmarshalYAML(u func(interface{}, bool) error) (_ error) {
+
+	var s string
+	err := u(&s, false)
+	if err != nil {
+		return err
+	}
+
+	genericErr := func(err error) error {
+		var buf strings.Builder
+		fmt.Fprintf(&buf, "cannot parse %q using regex %s", s, datarateRegex)
+		if err != nil {
+			fmt.Fprintf(&buf, ": %s", err)
+		}
+		return errors.New(buf.String())
+	}
+
+	match := datarateRegex.FindStringSubmatch(s)
+	if match == nil {
+		return genericErr(nil)
+	}
+
+	bps, err := strconv.ParseFloat(match[1], 64)
+	if err != nil {
+		return genericErr(err)
+	}
+
+	if match[2] == "bit" {
+		if math.Round(bps) != bps {
+			return genericErr(fmt.Errorf("unit bit must be an integer value"))
+		}
+		r.bits = bps
+		return nil
+	}
+
+	factorMap := map[string]uint64{
+		"": 1,
+
+		"K": 1e3,
+		"M": 1e6,
+		"G": 1e9,
+		"T": 1e12,
+
+		"Ki": 1 << 10,
+		"Mi": 1 << 20,
+		"Gi": 1 << 30,
+		"Ti": 1 << 40,
+	}
+	factor, ok := factorMap[match[3]]
+	if !ok {
+		panic(match)
+	}
+
+	baseUnitFactorMap := map[string]uint64{
+		"b": 1,
+		"B": 8,
+	}
+	baseUnitFactor, ok := baseUnitFactorMap[match[4]]
+	if !ok {
+		panic(match)
+	}
+
+	r.bits = bps * float64(factor) * float64(baseUnitFactor)
+	return nil
+}

internal/util/datasizeunit/datasizeunit_test.go

@@ -0,0 +1,57 @@
+package datasizeunit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestBits(t *testing.T) {
+
+	tcs := []struct {
+		input      string
+		expectRate float64
+		expectErr  string
+	}{
+		{`23 bit`, 23, ""}, // bit special case works
+		{`23bit`, 23, ""},  // also without space
+
+		{`10MiB`, 10 * (1 << 20) * 8, ""},  // integer unit without space
+		{`10 MiB`, 8 * 10 * (1 << 20), ""}, // integer unit with space
+
+		{`10.5 Kib`, 10.5 * (1 << 10), ""}, // floating point with bit unit works with space
+		{`10.5Kib`, 10.5 * (1 << 10), ""},  // floating point with bit unit works without space
+
+		// unit checks
+		{`1 bit`, 1, ""},
+		{`1 B`, 1 * 8, ""},
+		{`1 Kb`, 1e3, ""},
+		{`1 Kib`, 1 << 10, ""},
+		{`1 Mb`, 1e6, ""},
+		{`1 Mib`, 1 << 20, ""},
+		{`1 Gb`, 1e9, ""},
+		{`1 Gib`, 1 << 30, ""},
+		{`1 Tb`, 1e12, ""},
+		{`1 Tib`, 1 << 40, ""},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.input, func(t *testing.T) {
+
+			var bits Bits
+			err := yaml.Unmarshal([]byte(tc.input), &bits)
+			if tc.expectErr != "" {
+				assert.Error(t, err)
+				assert.Regexp(t, tc.expectErr, err.Error())
+				assert.Zero(t, bits.bits)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectRate, bits.bits)
+			}
+		})
+
+	}
+
+}

internal/util/devnoop/devnoop.go

@@ -0,0 +1,14 @@
+// package devnoop provides an io.ReadWriteCloser that never errors
+// and always reports reads / writes to / from buffers as complete.
+// The buffers themselves are never touched.
+package devnoop
+
+type Dev struct{}
+
+func Get() Dev {
+	return Dev{}
+}
+
+func (Dev) Write(p []byte) (n int, err error) { return len(p), nil }
+func (Dev) Read(p []byte) (n int, err error)  { return len(p), nil }
+func (Dev) Close() error                      { return nil }

internal/util/envconst/envconst.go

@@ -0,0 +1,176 @@
+package envconst
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"reflect"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var cache sync.Map
+
+func Reset() {
+	cache.Range(func(key, _ interface{}) bool {
+		cache.Delete(key)
+		return true
+	})
+}
+
+func Duration(varname string, def time.Duration) (d time.Duration) {
+	var err error
+	if v, ok := cache.Load(varname); ok {
+		return v.(time.Duration)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d, err = time.ParseDuration(e)
+		if err != nil {
+			panic(err)
+		}
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func Int(varname string, def int) (d int) {
+	if v, ok := cache.Load(varname); ok {
+		return v.(int)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d64, err := strconv.ParseInt(e, 10, strconv.IntSize)
+		if err != nil {
+			panic(err)
+		}
+		d = int(d64)
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func Int64(varname string, def int64) (d int64) {
+	var err error
+	if v, ok := cache.Load(varname); ok {
+		return v.(int64)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d, err = strconv.ParseInt(e, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func Uint64(varname string, def uint64) (d uint64) {
+	var err error
+	if v, ok := cache.Load(varname); ok {
+		return v.(uint64)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d, err = strconv.ParseUint(e, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func Bool(varname string, def bool) (d bool) {
+	var err error
+	if v, ok := cache.Load(varname); ok {
+		return v.(bool)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d, err = strconv.ParseBool(e)
+		if err != nil {
+			panic(err)
+		}
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func String(varname string, def string) (d string) {
+	if v, ok := cache.Load(varname); ok {
+		return v.(string)
+	}
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		d = e
+	}
+	cache.Store(varname, d)
+	return d
+}
+
+func Var(varname string, def flag.Value) (d interface{}) {
+
+	// use def's type to instantiate a new object of that same type
+	// and call flag.Value.Set() on it
+	defType := reflect.TypeOf(def)
+	if defType.Kind() != reflect.Ptr {
+		panic(fmt.Sprintf("envconst var must be a pointer, got %T", def))
+	}
+	defElemType := defType.Elem()
+
+	if v, ok := cache.Load(varname); ok {
+		return v
+	}
+
+	e := os.Getenv(varname)
+	if e == "" {
+		d = def
+	} else {
+		newInstance := reflect.New(defElemType)
+		if err := newInstance.Interface().(flag.Value).Set(e); err != nil {
+			panic(err)
+		}
+		d = newInstance.Interface()
+	}
+
+	cache.Store(varname, d)
+	return d
+}
+
+type Report struct {
+	Entries []EntryReport
+}
+
+type EntryReport struct {
+	Var         string
+	Value       string
+	ValueGoType string
+}
+
+func GetReport() *Report {
+	var r Report
+	cache.Range(func(key, value interface{}) bool {
+		r.Entries = append(r.Entries, EntryReport{
+			Var:         key.(string),
+			Value:       fmt.Sprintf("%v", value),
+			ValueGoType: fmt.Sprintf("%T", value),
+		})
+		return true
+	})
+	return &r
+}

internal/util/envconst/envconst_test.go

@@ -0,0 +1,58 @@
+package envconst_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+type ExampleVarType struct{ string }
+
+var (
+	Var1 = ExampleVarType{"var1"}
+	Var2 = ExampleVarType{"var2"}
+)
+
+func (m ExampleVarType) String() string { return string(m.string) }
+func (m *ExampleVarType) Set(s string) error {
+	switch s {
+	case Var1.String():
+		*m = Var1
+	case Var2.String():
+		*m = Var2
+	default:
+		return fmt.Errorf("unknown var %q", s)
+	}
+	return nil
+}
+
+const EnvVarName = "ZREPL_ENVCONST_UNIT_TEST_VAR"
+
+func TestVarDefaultValue(t *testing.T) {
+	envconst.Reset()
+	_, set := os.LookupEnv(EnvVarName)
+	require.False(t, set)
+	defer os.Unsetenv(EnvVarName)
+
+	val := envconst.Var(EnvVarName, &Var1)
+	if &Var1 != val {
+		t.Errorf("default value should be same address")
+	}
+}
+
+func TestVarOverriddenValue(t *testing.T) {
+	envconst.Reset()
+	_, set := os.LookupEnv(EnvVarName)
+	require.False(t, set)
+	defer os.Unsetenv(EnvVarName)
+
+	err := os.Setenv(EnvVarName, "var2")
+	require.NoError(t, err)
+
+	val := envconst.Var(EnvVarName, &Var1)
+	require.Equal(t, &Var2, val, "only structural identity is required for non-default vars")
+}

internal/util/errorarray/errorarray.go

@@ -0,0 +1,43 @@
+package errorarray
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Errors struct {
+	Msg     string
+	Wrapped []error
+}
+
+var _ error = (*Errors)(nil)
+
+func Wrap(errs []error, msg string) Errors {
+	if len(errs) == 0 {
+		panic("passing empty errs argument")
+	}
+	return Errors{Msg: msg, Wrapped: errs}
+}
+
+func (e Errors) Unwrap() error {
+	if len(e.Wrapped) == 1 {
+		return e.Wrapped[0]
+	}
+	return nil // ... limitation of the Go 1.13 errors API
+}
+
+func (e Errors) Error() string {
+	if len(e.Wrapped) == 1 {
+		return fmt.Sprintf("%s: %s", e.Msg, e.Wrapped[0])
+	}
+	var buf strings.Builder
+	fmt.Fprintf(&buf, "%s: multiple errors:\n", e.Msg)
+	for i, err := range e.Wrapped {
+		fmt.Fprintf(&buf, "%s", err)
+		if i != len(e.Wrapped)-1 {
+			fmt.Fprintf(&buf, "\n")
+		}
+	}
+	return buf.String()
+
+}

internal/util/iocommand/iocommand.go

@@ -0,0 +1,157 @@
+package iocommand
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+// An IOCommand exposes a forked process's std(in|out|err) through the io.ReadWriteCloser interface.
+type IOCommand struct {
+	Cmd        *exec.Cmd
+	kill       context.CancelFunc
+	Stdin      io.WriteCloser
+	Stdout     io.ReadCloser
+	StderrBuf  *bytes.Buffer
+	ExitResult *IOCommandExitResult
+}
+
+const IOCommandStderrBufSize = 1024
+
+type IOCommandError struct {
+	WaitErr error
+	Stderr  []byte
+}
+
+type IOCommandExitResult struct {
+	Error      error
+	WaitStatus syscall.WaitStatus
+}
+
+func (e IOCommandError) Error() string {
+	return fmt.Sprintf("underlying process exited with error: %s\nstderr: %s\n", e.WaitErr, e.Stderr)
+}
+
+func RunIOCommand(ctx context.Context, command string, args ...string) (c *IOCommand, err error) {
+	c, err = NewIOCommand(ctx, command, args, IOCommandStderrBufSize)
+	if err != nil {
+		return
+	}
+	err = c.Start()
+	return
+}
+
+func NewIOCommand(ctx context.Context, command string, args []string, stderrBufSize int) (c *IOCommand, err error) {
+
+	if stderrBufSize == 0 {
+		stderrBufSize = IOCommandStderrBufSize
+	}
+
+	c = &IOCommand{}
+
+	ctx, c.kill = context.WithCancel(ctx)
+	c.Cmd = exec.CommandContext(ctx, command, args...)
+
+	if c.Stdout, err = c.Cmd.StdoutPipe(); err != nil {
+		return
+	}
+
+	if c.Stdin, err = c.Cmd.StdinPipe(); err != nil {
+		return
+	}
+
+	c.StderrBuf = bytes.NewBuffer(make([]byte, 0, stderrBufSize))
+	c.Cmd.Stderr = c.StderrBuf
+
+	return
+
+}
+
+func (c *IOCommand) Start() (err error) {
+	if err = c.Cmd.Start(); err != nil {
+		return
+	}
+	return
+}
+
+// Read from process's stdout.
+// The behavior after Close()ing is undefined
+func (c *IOCommand) Read(buf []byte) (n int, err error) {
+	n, err = c.Stdout.Read(buf)
+	if err == io.EOF {
+		if waitErr := c.doWait(context.Background()); waitErr != nil {
+			err = waitErr
+		}
+	}
+	return
+}
+
+func (c *IOCommand) doWait(ctx context.Context) (err error) {
+	go func() {
+		dl, ok := ctx.Deadline()
+		if !ok {
+			return
+		}
+		time.Sleep(time.Until(dl))
+		c.kill()
+		c.Stdout.Close()
+		c.Stdin.Close()
+	}()
+	waitErr := c.Cmd.Wait()
+	var wasUs bool = false
+	var waitStatus syscall.WaitStatus
+	if c.Cmd.ProcessState == nil {
+		fmt.Fprintf(os.Stderr, "util.IOCommand: c.Cmd.ProcessState is nil after c.Cmd.Wait()\n")
+	}
+	if c.Cmd.ProcessState != nil {
+		sysSpecific := c.Cmd.ProcessState.Sys()
+		var ok bool
+		waitStatus, ok = sysSpecific.(syscall.WaitStatus)
+		if !ok {
+			fmt.Fprintf(os.Stderr, "util.IOCommand: c.Cmd.ProcessState.Sys() could not be converted to syscall.WaitStatus: %T\n", sysSpecific)
+			os.Stderr.Sync()
+			panic(sysSpecific) // this can only be true if we are not on UNIX, and we don't support that
+		}
+		wasUs = waitStatus.Signaled() && waitStatus.Signal() == syscall.SIGTERM // in Close()
+	}
+
+	if waitErr != nil && !wasUs {
+		err = IOCommandError{
+			WaitErr: waitErr,
+			Stderr:  c.StderrBuf.Bytes(),
+		}
+	}
+
+	c.ExitResult = &IOCommandExitResult{
+		Error:      err, // is still empty if waitErr was due to signalling
+		WaitStatus: waitStatus,
+	}
+	return
+}
+
+// Write to process's stdin.
+// The behavior after Close()ing is undefined
+func (c *IOCommand) Write(buf []byte) (n int, err error) {
+	return c.Stdin.Write(buf)
+}
+
+// Terminate the child process and collect its exit status
+// It is safe to call Close() multiple times.
+func (c *IOCommand) Close() (err error) {
+	if c.Cmd.ProcessState == nil {
+		// racy...
+		err = syscall.Kill(c.Cmd.Process.Pid, syscall.SIGTERM)
+		ctx, cancel := context.WithTimeout(context.Background(), envconst.Duration("IOCOMMAND_TIMEOUT", 10*time.Second))
+		defer cancel()
+		return c.doWait(ctx)
+	} else {
+		return c.ExitResult.Error
+	}
+}

internal/util/limitio/limitio.go

@@ -0,0 +1,36 @@
+package limitio
+
+import "io"
+
+type readCloser struct {
+	read  int64
+	limit int64
+	r     io.ReadCloser
+}
+
+func ReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser {
+	return &readCloser{0, limit, rc}
+}
+
+var _ io.ReadCloser = (*readCloser)(nil)
+
+func (r *readCloser) Read(b []byte) (int, error) {
+
+	if len(b) == 0 {
+		return 0, nil
+	}
+
+	if r.read == r.limit {
+		return 0, io.EOF
+	}
+
+	if r.read+int64(len(b)) >= r.limit {
+		b = b[:int(r.limit-r.read)]
+	}
+
+	readN, err := r.r.Read(b)
+	r.read += int64(readN)
+	return readN, err
+}
+
+func (r *readCloser) Close() error { return r.r.Close() }

internal/util/limitio/limitio_readcloser_test.go

@@ -0,0 +1,44 @@
+package limitio
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockRC struct {
+	r      io.Reader
+	closed bool
+}
+
+func newMockRC(r io.Reader) *mockRC { return &mockRC{r, false} }
+
+func (m mockRC) Read(b []byte) (int, error) {
+	if m.closed {
+		return 0, io.ErrClosedPipe
+	}
+	return m.r.Read(b)
+}
+
+func (m *mockRC) Close() error {
+	m.closed = true
+	return nil
+}
+
+func TestReadCloser(t *testing.T) {
+	foobarReader := bytes.NewReader([]byte("foobar2342"))
+	mock := newMockRC(foobarReader)
+	limited := ReadCloser(mock, 6)
+	var buf [20]byte
+	n, err := limited.Read(buf[:])
+	assert.NoError(t, err)
+	assert.Equal(t, 6, n)
+	assert.Equal(t, buf[:n], []byte("foobar"))
+	n, err = limited.Read(buf[:])
+	assert.Equal(t, 0, n)
+	assert.Equal(t, io.EOF, err)
+	limited.Close()
+	assert.True(t, mock.closed)
+}

internal/util/nodefault/nodefault.go

@@ -0,0 +1,31 @@
+// Package nodefault provides newtypes around builtin Go types
+// so that if the newtype is used as a field in a struct and that field
+// is zero-initialized (https://golang.org/ref/spec#The_zero_value),
+// accessing the field will cause a null pointer deref panic.
+// Or in other terms: It soft-enforces that the caller sets the field
+// explicitly when constructing the struct.
+//
+// Example:
+//
+//	type Config struct {
+//	    // This field must be set to a non-nil value,
+//	    // forcing the caller to make their mind up
+//	    // about this field.
+//	    CriticalSetting   *nodefault.Bool
+//	}
+//
+// An function that takes such a Config should _not_ check for nil-ness:
+// and instead unconditionally dereference:
+//
+//	func f(c Config) {
+//	    if (c.CriticalSetting) { }
+//	}
+//
+// If the caller of f forgot to specify the .CriticalSetting
+// field, the Go runtime will issue a nil-pointer deref panic
+// and it'll be clear that the caller did not read the docs of Config.
+//
+//	f(Config{}) // crashes
+//
+//	f Config{ CriticalSetting: &nodefault.Bool{B: false}} // doesn't crash
+package nodefault

internal/util/nodefault/nodefault_bool.go

@@ -0,0 +1,19 @@
+package nodefault
+
+import "fmt"
+
+type Bool struct{ B bool }
+
+func (n *Bool) ValidateNoDefault() error {
+	if n == nil {
+		return fmt.Errorf("must explicitly set `true` or `false`")
+	}
+	return nil
+}
+
+func (n *Bool) String() string {
+	if n == nil {
+		return "unset"
+	}
+	return fmt.Sprintf("%v", n.B)
+}

internal/util/optionaldeadline/optionaldeadline.go

@@ -0,0 +1,83 @@
+package optionaldeadline
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+type contextWithOptionalDeadline struct {
+	context.Context
+
+	m        sync.Mutex
+	deadline time.Time
+
+	done chan struct{}
+	err  error
+}
+
+func (c *contextWithOptionalDeadline) Deadline() (deadline time.Time, ok bool) {
+	c.m.Lock()
+	defer c.m.Unlock()
+	return c.deadline, !c.deadline.IsZero()
+}
+
+func (c *contextWithOptionalDeadline) Err() error {
+	c.m.Lock()
+	defer c.m.Unlock()
+	return c.err
+}
+
+func (c *contextWithOptionalDeadline) Done() <-chan struct{} {
+	return c.done
+}
+
+func ContextWithOptionalDeadline(pctx context.Context) (ctx context.Context, enforceDeadline func(deadline time.Time)) {
+
+	// mctx can only be cancelled by cancelMctx, not by a potential cancel of pctx
+	rctx := &contextWithOptionalDeadline{
+		Context: pctx,
+		done:    make(chan struct{}),
+		err:     nil,
+	}
+	enforceDeadline = func(deadline time.Time) {
+
+		// Set deadline and prohibit multiple calls
+		rctx.m.Lock()
+		alreadyCalled := !rctx.deadline.IsZero()
+		if !alreadyCalled {
+			rctx.deadline = deadline
+		}
+		rctx.m.Unlock()
+		if alreadyCalled {
+			return
+		}
+
+		// Deadline in past?
+		sleepTime := time.Until(deadline)
+		if sleepTime <= 0 {
+			rctx.m.Lock()
+			rctx.err = context.DeadlineExceeded
+			rctx.m.Unlock()
+			close(rctx.done)
+			return
+		}
+		go func() {
+			// Set a timer and wait for timer or parent context to be cancelled
+			timer := time.NewTimer(sleepTime)
+			var setErr error
+			select {
+			case <-pctx.Done():
+				timer.Stop()
+				setErr = pctx.Err()
+			case <-timer.C:
+				setErr = context.DeadlineExceeded
+			}
+			rctx.m.Lock()
+			rctx.err = setErr
+			rctx.m.Unlock()
+			close(rctx.done)
+		}()
+	}
+	return rctx, enforceDeadline
+}

internal/util/optionaldeadline/optionaldeadline_test.go

@@ -0,0 +1,107 @@
+package optionaldeadline
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/util/chainlock"
+	"github.com/zrepl/zrepl/internal/util/zreplcircleci"
+)
+
+func TestContextWithOptionalDeadline(t *testing.T) {
+	zreplcircleci.SkipOnCircleCI(t, "test relies on predictably low scheduler latency")
+
+	ctx := context.Background()
+	cctx, enforceDeadline := ContextWithOptionalDeadline(ctx)
+
+	begin := time.Now()
+	var checker struct {
+		receivedCancellation time.Time
+		cancellationError    error
+		timeout              bool
+		mtx                  chainlock.L
+	}
+	go func() {
+		select {
+		case <-cctx.Done():
+			defer checker.mtx.Lock().Unlock()
+			checker.receivedCancellation = time.Now()
+			checker.cancellationError = cctx.Err()
+		case <-time.After(600 * time.Millisecond):
+			defer checker.mtx.Lock().Unlock()
+			checker.timeout = true
+		}
+	}()
+	defer checker.mtx.Lock().Unlock()
+	checker.mtx.DropWhile(func() {
+		time.Sleep(100 * time.Millisecond)
+	})
+	if !checker.receivedCancellation.IsZero() {
+		t.Fatalf("no enforcement means no cancellation")
+	}
+	require.Nil(t, cctx.Err(), "no error while not cancelled")
+	dl, ok := cctx.Deadline()
+	require.False(t, ok)
+	require.Zero(t, dl)
+	enforceDeadline(begin.Add(200 * time.Millisecond))
+	// second call must be ignored, i.e. we expect the deadline to be at begin+200ms, not begin+400ms
+	enforceDeadline(begin.Add(400 * time.Millisecond))
+
+	checker.mtx.DropWhile(func() {
+		time.Sleep(300 * time.Millisecond) // 100ms margin for scheduler
+	})
+	assert.False(t, checker.timeout, "test timeout")
+	receivedCancellationAfter := checker.receivedCancellation.Sub(begin)
+	if receivedCancellationAfter > 250*time.Millisecond {
+		t.Fatalf("cancellation is beyond acceptable scheduler latency: %s", receivedCancellationAfter)
+	}
+	require.Equal(t, context.DeadlineExceeded, checker.cancellationError)
+}
+
+func TestContextWithOptionalDeadlineNegativeDeadline(t *testing.T) {
+	ctx := context.Background()
+	cctx, enforceDeadline := ContextWithOptionalDeadline(ctx)
+	enforceDeadline(time.Now().Add(-10 * time.Second))
+	select {
+	case <-cctx.Done():
+	default:
+		t.FailNow()
+	}
+}
+
+func TestContextWithOptionalDeadlineParentCancellation(t *testing.T) {
+	zreplcircleci.SkipOnCircleCI(t, "test relies on predictably low scheduler latency")
+
+	pctx, cancel := context.WithCancel(context.Background())
+	cctx, enforceDeadline := ContextWithOptionalDeadline(pctx)
+
+	// 0 ms
+	start := time.Now()
+	enforceDeadline(start.Add(400 * time.Millisecond))
+	time.Sleep(100 * time.Millisecond)
+	cancel()                           // cancel @ ~100ms
+	time.Sleep(100 * time.Millisecond) // give 100ms time to propagate cancel
+	// @ ~200ms
+	select {
+	case <-cctx.Done():
+		assert.True(t, time.Now().Before(start.Add(300*time.Millisecond)))
+		assert.Equal(t, context.Canceled, cctx.Err())
+	default:
+		t.FailNow()
+	}
+
+}
+
+type testContextKey string
+
+const testContextKeyKey testContextKey = "key"
+
+func TestContextWithOptionalDeadlineValue(t *testing.T) {
+	pctx := context.WithValue(context.Background(), testContextKeyKey, "value")
+	cctx, _ := ContextWithOptionalDeadline(pctx)
+	assert.Equal(t, "value", cctx.Value(testContextKeyKey))
+}

internal/util/semaphore/semaphore.go

@@ -0,0 +1,41 @@
+package semaphore
+
+import (
+	"context"
+
+	wsemaphore "golang.org/x/sync/semaphore"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+)
+
+type S struct {
+	ws *wsemaphore.Weighted
+}
+
+func New(max int64) *S {
+	return &S{wsemaphore.NewWeighted(max)}
+}
+
+type AcquireGuard struct {
+	s        *S
+	released bool
+}
+
+// The returned AcquireGuard is not goroutine-safe.
+func (s *S) Acquire(ctx context.Context) (*AcquireGuard, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+	if err := s.ws.Acquire(ctx, 1); err != nil {
+		return nil, err
+	} else if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	return &AcquireGuard{s, false}, nil
+}
+
+func (g *AcquireGuard) Release() {
+	if g == nil || g.released {
+		return
+	}
+	g.released = true
+	g.s.ws.Release(1)
+}

internal/util/semaphore/semaphore_test.go

@@ -0,0 +1,52 @@
+package semaphore
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+)
+
+func TestSemaphore(t *testing.T) {
+	const numGoroutines = 10
+	const concurrentSemaphore = 6
+	const sleepTime = 1 * time.Second
+
+	begin := time.Now()
+
+	sem := New(concurrentSemaphore)
+
+	var acquisitions struct {
+		beforeT, afterT uint32
+	}
+
+	rootCtx, endRoot := trace.WithTaskFromStack(context.Background())
+	defer endRoot()
+	_, add, waitEnd := trace.WithTaskGroup(rootCtx, "TestSemaphore")
+
+	for i := 0; i < numGoroutines; i++ {
+		// not capturing i so no need for local copy
+		add(func(ctx context.Context) {
+			res, err := sem.Acquire(ctx)
+			require.NoError(t, err)
+			defer res.Release()
+			if time.Since(begin) > sleepTime {
+				atomic.AddUint32(&acquisitions.afterT, 1)
+			} else {
+				atomic.AddUint32(&acquisitions.beforeT, 1)
+			}
+			time.Sleep(sleepTime)
+		})
+	}
+
+	waitEnd()
+
+	assert.True(t, acquisitions.beforeT == concurrentSemaphore)
+	assert.True(t, acquisitions.afterT == numGoroutines-concurrentSemaphore)
+
+}

internal/util/socketpair/socketpair.go

@@ -0,0 +1,39 @@
+package socketpair
+
+import (
+	"net"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func SocketPair() (a, b *net.UnixConn, err error) {
+	// don't use net.Pipe, as it doesn't implement things like lingering, which our code relies on
+	sockpair, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	toConn := func(fd int) (*net.UnixConn, error) {
+		f := os.NewFile(uintptr(fd), "fileconn")
+		if f == nil {
+			panic(fd)
+		}
+		c, err := net.FileConn(f)
+		f.Close() // net.FileConn uses dup under the hood
+		if err != nil {
+			return nil, err
+		}
+		// strictly, the following type assertion is an implementation detail
+		// however, will be caught by test TestSocketPairWorks
+		fileConnIsUnixConn := c.(*net.UnixConn)
+		return fileConnIsUnixConn, nil
+	}
+	if a, err = toConn(sockpair[0]); err != nil { // shadowing
+		return nil, nil, err
+	}
+	if b, err = toConn(sockpair[1]); err != nil { // shadowing
+		a.Close()
+		return nil, nil, err
+	}
+	return a, b, nil
+}

internal/util/socketpair/socketpair_test.go

@@ -0,0 +1,18 @@
+package socketpair
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// This is test is mostly to verify that the assumption about
+// net.FileConn returning *net.UnixConn for AF_UNIX FDs works.
+func TestSocketPairWorks(t *testing.T) {
+	assert.NotPanics(t, func() {
+		a, b, err := SocketPair()
+		assert.NoError(t, err)
+		a.Close()
+		b.Close()
+	})
+}

internal/util/suspendresumesafetimer/suspendresumesafetimer.go

@@ -0,0 +1,96 @@
+package suspendresumesafetimer
+
+import (
+	"context"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+// The returned error is guaranteed to be the ctx.Err()
+func SleepUntil(ctx context.Context, sleepUntil time.Time) error {
+
+	// We use .Round(0) to strip the monotonic clock reading from the time.Time
+	// returned by time.Now(). That will make the before/after check in the ticker
+	// for-loop compare wall-clock times instead of monotonic time.
+	// Comparing wall clock time is necessary because monotonic time does not progress
+	// while the system is suspended.
+	//
+	// Background
+	//
+	// A time.Time carries a wallclock timestamp and optionally a monotonic clock timestamp.
+	// time.Now() returns a time.Time that carries both.
+	// time.Time.Add() applies the same delta to both timestamps in the time.Time.
+	// x.Sub(y) will return the *monotonic* delta if both x and y carry a monotonic timestamp.
+	// time.Until(x) == x.Sub(now) where `now` will have a monotonic timestamp.
+	// So, time.Until(x) with an `x` that has monotonic timestamp will return monotonic delta.
+	//
+	// Why Do We Care?
+	//
+	// On systems that suspend/resume, wall clock time progresses during suspend but
+	// monotonic time does not.
+	//
+	// So, suppose the following sequence of events:
+	//   x <== time.Now()
+	//   System suspends for 1 hour
+	//   delta <== time.Now().Sub(x)
+	// `delta` will be near 0 because time.Until() subtracts the monotonic
+	// timestamps, and monotonic time didn't progress during suspend.
+	//
+	// Now strip the timestamp using .Round(0)
+	//   x <== time.Now().Round(0)
+	//   System suspends for 1 hour
+	//   delta <== time.Now().Sub(x)
+	// `delta` will be 1 hour because time.Sub() subtracted wallclock timestamps
+	// because x didn't have a monotonic timestamp because we stripped it using .Round(0).
+	//
+	//
+	sleepUntil = sleepUntil.Round(0)
+
+	// Set up a timer so that, if the system doesn't suspend/resume,
+	// we get a precise wake-up time from the native Go timer.
+	monotonicClockTimer := time.NewTimer(time.Until(sleepUntil))
+	defer func() {
+		if !monotonicClockTimer.Stop() {
+			// non-blocking read since we can come here when
+			// we've already drained the channel through
+			//		case <-monotonicClockTimer.C
+			// in the `for` loop below.
+			select {
+			case <-monotonicClockTimer.C:
+			default:
+			}
+		}
+	}()
+
+	// Set up a ticker so that we're guaranteed to wake up periodically.
+	// We'll then get the current wall-clock time and check ourselves
+	// whether we're past the requested expiration time.
+	// Pick a 10 second check interval by default since it's rare that
+	// suspend/resume is done more frequently.
+	ticker := time.NewTicker(envconst.Duration("ZREPL_WALLCLOCKTIMER_MAX_DELAY", 10*time.Second))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-monotonicClockTimer.C:
+			return nil
+		case <-ticker.C:
+			now := time.Now()
+			if now.Before(sleepUntil) {
+				// Continue waiting.
+
+				// Reset the monotonic timer to reset drift.
+				if !monotonicClockTimer.Stop() {
+					<-monotonicClockTimer.C
+				}
+				monotonicClockTimer.Reset(time.Until(sleepUntil))
+
+				continue
+			}
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}

internal/util/tcpsock/tcpsock.go

@@ -0,0 +1,27 @@
+package tcpsock
+
+import (
+	"context"
+	"net"
+	"syscall"
+)
+
+func Listen(address string, tryFreeBind bool) (*net.TCPListener, error) {
+	control := func(network, address string, c syscall.RawConn) error {
+		if tryFreeBind {
+			if err := freeBind(network, address, c); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	var listenConfig = net.ListenConfig{
+		Control: control,
+	}
+
+	l, err := listenConfig.Listen(context.Background(), "tcp", address)
+	if err != nil {
+		return nil, err
+	}
+	return l.(*net.TCPListener), nil
+}

internal/util/tcpsock/tcpsock_freebind_freebsd.go

@@ -0,0 +1,26 @@
+//go:build freebsd
+// +build freebsd
+
+package tcpsock
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func freeBind(network, address string, c syscall.RawConn) error {
+	var err, sockerr error
+	err = c.Control(func(fd uintptr) {
+		if network == "tcp6" {
+			sockerr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IPV6, syscall.IPV6_BINDANY, 1)
+		} else if network == "tcp4" {
+			sockerr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IP, syscall.IP_BINDANY, 1)
+		} else {
+			sockerr = fmt.Errorf("expecting 'tcp6' or 'tcp4', got %q", network)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return sockerr
+}

internal/util/tcpsock/tcpsock_freebind_linux.go

@@ -0,0 +1,20 @@
+//go:build linux
+// +build linux
+
+package tcpsock
+
+import (
+	"syscall"
+)
+
+func freeBind(network, address string, c syscall.RawConn) error {
+	var err, sockerr error
+	err = c.Control(func(fd uintptr) {
+		// apparently, this works for both IPv4 and IPv6
+		sockerr = syscall.SetsockoptInt(int(fd), syscall.SOL_IP, syscall.IP_FREEBIND, 1)
+	})
+	if err != nil {
+		return err
+	}
+	return sockerr
+}

internal/util/tcpsock/tcpsock_freebind_unsupported.go

@@ -0,0 +1,13 @@
+//go:build !linux && !freebsd
+// +build !linux,!freebsd
+
+package tcpsock
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func freeBind(network, address string, c syscall.RawConn) error {
+	return fmt.Errorf("IP_FREEBIND equivalent functionality not supported  on this platform")
+}

internal/util/zreplcircleci/zreplcircleci.go

@@ -0,0 +1,13 @@
+package zreplcircleci
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func SkipOnCircleCI(t *testing.T, reasonFmt string, args ...interface{}) {
+	if os.Getenv("CIRCLECI") != "" {
+		t.Skipf("This test is skipped in CircleCI. Reason: %s", fmt.Sprintf(reasonFmt, args...))
+	}
+}