move implementation to internal/ directory (#828)
This commit is contained in:
Christian Schwarz
committed by
GitHub
GitHub
parent
b9b9ad10cf
commit
908807bd59
@@ -0,0 +1,567 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/replication"
|
||||
"github.com/zrepl/zrepl/internal/replication/driver"
|
||||
"github.com/zrepl/zrepl/internal/replication/logic"
|
||||
"github.com/zrepl/zrepl/internal/replication/report"
|
||||
"github.com/zrepl/zrepl/internal/rpc"
|
||||
"github.com/zrepl/zrepl/internal/transport"
|
||||
"github.com/zrepl/zrepl/internal/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
)
|
||||
|
||||
type ActiveSide struct {
|
||||
mode activeMode
|
||||
name endpoint.JobID
|
||||
connecter transport.Connecter
|
||||
|
||||
replicationDriverConfig driver.Config
|
||||
|
||||
prunerFactory *pruner.PrunerFactory
|
||||
|
||||
promRepStateSecs *prometheus.HistogramVec // labels: state
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
||||
promReplicationErrors prometheus.Gauge
|
||||
promLastSuccessful prometheus.Gauge
|
||||
|
||||
tasksMtx sync.Mutex
|
||||
tasks activeSideTasks
|
||||
}
|
||||
|
||||
//go:generate enumer -type=ActiveSideState
|
||||
type ActiveSideState int
|
||||
|
||||
const (
|
||||
ActiveSideReplicating ActiveSideState = 1 << iota
|
||||
ActiveSidePruneSender
|
||||
ActiveSidePruneReceiver
|
||||
ActiveSideDone // also errors
|
||||
)
|
||||
|
||||
type activeSideTasks struct {
|
||||
state ActiveSideState
|
||||
|
||||
// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||
replicationReport driver.ReportFunc
|
||||
replicationCancel context.CancelFunc
|
||||
|
||||
// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||
prunerSender, prunerReceiver *pruner.Pruner
|
||||
|
||||
// valid for state ActiveSidePruneReceiver, ActiveSideDone
|
||||
prunerSenderCancel, prunerReceiverCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
|
||||
a.tasksMtx.Lock()
|
||||
defer a.tasksMtx.Unlock()
|
||||
copy := a.tasks
|
||||
if u == nil {
|
||||
return copy
|
||||
}
|
||||
u(©)
|
||||
a.tasks = copy
|
||||
return copy
|
||||
}
|
||||
|
||||
type activeMode interface {
|
||||
ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
|
||||
DisconnectEndpoints()
|
||||
SenderReceiver() (logic.Sender, logic.Receiver)
|
||||
Type() Type
|
||||
PlannerPolicy() logic.PlannerPolicy
|
||||
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
|
||||
SnapperReport() *snapper.Report
|
||||
ResetConnectBackoff()
|
||||
}
|
||||
|
||||
type modePush struct {
|
||||
setupMtx sync.Mutex
|
||||
sender *endpoint.Sender
|
||||
receiver *rpc.Client
|
||||
senderConfig *endpoint.SenderConfig
|
||||
plannerPolicy *logic.PlannerPolicy
|
||||
snapper snapper.Snapper
|
||||
}
|
||||
|
||||
func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil || m.sender != nil {
|
||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||
}
|
||||
m.sender = endpoint.NewSender(*m.senderConfig)
|
||||
m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
|
||||
}
|
||||
|
||||
func (m *modePush) DisconnectEndpoints() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
m.receiver.Close()
|
||||
m.sender = nil
|
||||
m.receiver = nil
|
||||
}
|
||||
|
||||
func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
return m.sender, m.receiver
|
||||
}
|
||||
|
||||
func (m *modePush) Type() Type { return TypePush }
|
||||
|
||||
func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
||||
|
||||
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
m.snapper.Run(ctx, wakeUpCommon)
|
||||
}
|
||||
|
||||
func (m *modePush) SnapperReport() *snapper.Report {
|
||||
r := m.snapper.Report()
|
||||
return &r
|
||||
}
|
||||
|
||||
func (m *modePush) ResetConnectBackoff() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil {
|
||||
m.receiver.ResetConnectBackoff()
|
||||
}
|
||||
}
|
||||
|
||||
func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
|
||||
m := &modePush{}
|
||||
var err error
|
||||
|
||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "sender config")
|
||||
}
|
||||
|
||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `replication`")
|
||||
}
|
||||
|
||||
conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `conflict_resolution`")
|
||||
}
|
||||
|
||||
m.plannerPolicy = &logic.PlannerPolicy{
|
||||
ConflictResolution: conflictResolution,
|
||||
ReplicationConfig: replicationConfig,
|
||||
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
|
||||
}
|
||||
if err := m.plannerPolicy.Validate(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build planner policy")
|
||||
}
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type modePull struct {
|
||||
setupMtx sync.Mutex
|
||||
receiver *endpoint.Receiver
|
||||
receiverConfig endpoint.ReceiverConfig
|
||||
sender *rpc.Client
|
||||
plannerPolicy *logic.PlannerPolicy
|
||||
interval config.PositiveDurationOrManual
|
||||
}
|
||||
|
||||
func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil || m.sender != nil {
|
||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||
}
|
||||
m.receiver = endpoint.NewReceiver(m.receiverConfig)
|
||||
m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
|
||||
}
|
||||
|
||||
func (m *modePull) DisconnectEndpoints() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
m.sender.Close()
|
||||
m.sender = nil
|
||||
m.receiver = nil
|
||||
}
|
||||
|
||||
func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
return m.sender, m.receiver
|
||||
}
|
||||
|
||||
func (*modePull) Type() Type { return TypePull }
|
||||
|
||||
func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
||||
|
||||
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
if m.interval.Manual {
|
||||
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
|
||||
// "waiting for wakeups" is printed in common ActiveSide.do
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(m.interval.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
select {
|
||||
case wakeUpCommon <- struct{}{}:
|
||||
default:
|
||||
GetLogger(ctx).
|
||||
WithField("pull_interval", m.interval).
|
||||
Warn("pull job took longer than pull interval")
|
||||
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *modePull) SnapperReport() *snapper.Report {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *modePull) ResetConnectBackoff() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.sender != nil {
|
||||
m.sender.ResetConnectBackoff()
|
||||
}
|
||||
}
|
||||
|
||||
func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
|
||||
m = &modePull{}
|
||||
m.interval = in.Interval
|
||||
|
||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `replication`")
|
||||
}
|
||||
|
||||
conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `conflict_resolution`")
|
||||
}
|
||||
|
||||
m.plannerPolicy = &logic.PlannerPolicy{
|
||||
ConflictResolution: conflictResolution,
|
||||
ReplicationConfig: replicationConfig,
|
||||
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
|
||||
}
|
||||
if err := m.plannerPolicy.Validate(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build planner policy")
|
||||
}
|
||||
|
||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func replicationDriverConfigFromConfig(in *config.Replication) (c driver.Config, err error) {
|
||||
c = driver.Config{
|
||||
StepQueueConcurrency: in.Concurrency.Steps,
|
||||
MaxAttempts: envconst.Int("ZREPL_REPLICATION_MAX_ATTEMPTS", 3),
|
||||
ReconnectHardFailTimeout: envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute),
|
||||
}
|
||||
err = c.Validate()
|
||||
return c, err
|
||||
}
|
||||
|
||||
func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}, parseFlags config.ParseFlags) (j *ActiveSide, err error) {
|
||||
|
||||
j = &ActiveSide{}
|
||||
j.name, err = endpoint.MakeJobID(in.Name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "invalid job name")
|
||||
}
|
||||
|
||||
switch v := configJob.(type) {
|
||||
case *config.PushJob:
|
||||
j.mode, err = modePushFromConfig(g, v, j.name) // shadow
|
||||
case *config.PullJob:
|
||||
j.mode, err = modePullFromConfig(g, v, j.name) // shadow
|
||||
default:
|
||||
panic(fmt.Sprintf("implementation error: unknown job type %T", v))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err // no wrapping required
|
||||
}
|
||||
|
||||
j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "state_time",
|
||||
Help: "seconds spent during replication",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
}, []string{"state"})
|
||||
j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "bytes_replicated",
|
||||
Help: "number of bytes replicated from sender to receiver per filesystem",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
}, []string{"filesystem"})
|
||||
j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "filesystem_errors",
|
||||
Help: "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
})
|
||||
j.promLastSuccessful = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "last_successful",
|
||||
Help: "timestamp of last successful replication",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
})
|
||||
|
||||
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect, parseFlags)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build client")
|
||||
}
|
||||
|
||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "pruning",
|
||||
Name: "time",
|
||||
Help: "seconds spent in pruner",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
}, []string{"prune_side"})
|
||||
j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j.replicationDriverConfig, err = replicationDriverConfigFromConfig(in.Replication)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build replication driver config")
|
||||
}
|
||||
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
registerer.MustRegister(j.promRepStateSecs)
|
||||
registerer.MustRegister(j.promPruneSecs)
|
||||
registerer.MustRegister(j.promBytesReplicated)
|
||||
registerer.MustRegister(j.promReplicationErrors)
|
||||
registerer.MustRegister(j.promLastSuccessful)
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Name() string { return j.name.String() }
|
||||
|
||||
type ActiveSideStatus struct {
|
||||
Replication *report.Report
|
||||
PruningSender, PruningReceiver *pruner.Report
|
||||
Snapshotting *snapper.Report
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Status() *Status {
|
||||
tasks := j.updateTasks(nil)
|
||||
|
||||
s := &ActiveSideStatus{}
|
||||
t := j.mode.Type()
|
||||
if tasks.replicationReport != nil {
|
||||
s.Replication = tasks.replicationReport()
|
||||
}
|
||||
if tasks.prunerSender != nil {
|
||||
s.PruningSender = tasks.prunerSender.Report()
|
||||
}
|
||||
if tasks.prunerReceiver != nil {
|
||||
s.PruningReceiver = tasks.prunerReceiver.Report()
|
||||
}
|
||||
s.Snapshotting = j.mode.SnapperReport()
|
||||
return &Status{Type: t, JobSpecific: s}
|
||||
}
|
||||
|
||||
func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
pull, ok := j.mode.(*modePull)
|
||||
if !ok {
|
||||
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
|
||||
return nil, false
|
||||
}
|
||||
return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
|
||||
}
|
||||
|
||||
func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
|
||||
push, ok := j.mode.(*modePush)
|
||||
if !ok {
|
||||
_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
|
||||
return nil
|
||||
}
|
||||
return push.senderConfig
|
||||
}
|
||||
|
||||
// The active side of a replication uses one end (sender or receiver)
|
||||
// directly by method invocation, without going through a transport that
|
||||
// provides a client identity.
|
||||
// However, in order to avoid the need to distinguish between direct-method-invocating
|
||||
// clients and RPC client, we use an invalid client identity as a sentinel value.
|
||||
func FakeActiveSideDirectMethodInvocationClientIdentity(jobId endpoint.JobID) string {
|
||||
return fmt.Sprintf("<local><active><job><client><identity><job=%q>", jobId.String())
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Run(ctx context.Context) {
|
||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
|
||||
defer endTask()
|
||||
|
||||
ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, FakeActiveSideDirectMethodInvocationClientIdentity(j.name))
|
||||
|
||||
log := GetLogger(ctx)
|
||||
|
||||
defer log.Info("job exiting")
|
||||
|
||||
periodicDone := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
periodicCtx, endTask := trace.WithTask(ctx, "periodic")
|
||||
defer endTask()
|
||||
go j.mode.RunPeriodic(periodicCtx, periodicDone)
|
||||
|
||||
invocationCount := 0
|
||||
outer:
|
||||
for {
|
||||
log.Info("wait for wakeups")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
|
||||
case <-wakeup.Wait(ctx):
|
||||
j.mode.ResetConnectBackoff()
|
||||
case <-periodicDone:
|
||||
}
|
||||
invocationCount++
|
||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
||||
j.do(invocationCtx)
|
||||
endSpan()
|
||||
}
|
||||
}
|
||||
|
||||
func (j *ActiveSide) do(ctx context.Context) {
|
||||
|
||||
j.mode.ConnectEndpoints(ctx, j.connecter)
|
||||
defer j.mode.DisconnectEndpoints()
|
||||
|
||||
// allow cancellation of an invocation (this function)
|
||||
ctx, cancelThisRun := context.WithCancel(ctx)
|
||||
defer cancelThisRun()
|
||||
go func() {
|
||||
select {
|
||||
case <-reset.Wait(ctx):
|
||||
GetLogger(ctx).Info("reset received, cancelling current invocation")
|
||||
cancelThisRun()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
sender, receiver := j.mode.SenderReceiver()
|
||||
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, endSpan := trace.WithSpan(ctx, "replication")
|
||||
ctx, repCancel := context.WithCancel(ctx)
|
||||
var repWait driver.WaitFunc
|
||||
j.updateTasks(func(tasks *activeSideTasks) {
|
||||
// reset it
|
||||
*tasks = activeSideTasks{}
|
||||
tasks.replicationCancel = func() { repCancel(); endSpan() }
|
||||
tasks.replicationReport, repWait = replication.Do(
|
||||
ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
|
||||
)
|
||||
tasks.state = ActiveSideReplicating
|
||||
})
|
||||
GetLogger(ctx).Info("start replication")
|
||||
repWait(true) // wait blocking
|
||||
repCancel() // always cancel to free up context resources
|
||||
|
||||
replicationReport := j.tasks.replicationReport()
|
||||
var numErrors = replicationReport.GetFailedFilesystemsCountInLatestAttempt()
|
||||
j.promReplicationErrors.Set(float64(numErrors))
|
||||
if numErrors == 0 {
|
||||
j.promLastSuccessful.SetToCurrentTime()
|
||||
}
|
||||
|
||||
endSpan()
|
||||
}
|
||||
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
|
||||
ctx, senderCancel := context.WithCancel(ctx)
|
||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
|
||||
tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
|
||||
tasks.state = ActiveSidePruneSender
|
||||
})
|
||||
GetLogger(ctx).Info("start pruning sender")
|
||||
tasks.prunerSender.Prune()
|
||||
GetLogger(ctx).Info("finished pruning sender")
|
||||
senderCancel()
|
||||
endSpan()
|
||||
}
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
|
||||
ctx, receiverCancel := context.WithCancel(ctx)
|
||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
|
||||
tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
|
||||
tasks.state = ActiveSidePruneReceiver
|
||||
})
|
||||
GetLogger(ctx).Info("start pruning receiver")
|
||||
tasks.prunerReceiver.Prune()
|
||||
GetLogger(ctx).Info("finished pruning receiver")
|
||||
receiverCancel()
|
||||
endSpan()
|
||||
}
|
||||
|
||||
j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.state = ActiveSideDone
|
||||
})
|
||||
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/transport"
|
||||
)
|
||||
|
||||
func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
|
||||
jobid, err := endpoint.MakeJobID("validjobname")
|
||||
require.NoError(t, err)
|
||||
clientIdentity := FakeActiveSideDirectMethodInvocationClientIdentity(jobid)
|
||||
t.Logf("%v", clientIdentity)
|
||||
err = transport.ValidateClientIdentity(clientIdentity)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
||||
|
||||
package job
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
|
||||
_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
|
||||
_ActiveSideStateName_2 = "ActiveSideDone"
|
||||
)
|
||||
|
||||
var (
|
||||
_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
|
||||
_ActiveSideStateIndex_1 = [...]uint8{0, 23}
|
||||
_ActiveSideStateIndex_2 = [...]uint8{0, 14}
|
||||
)
|
||||
|
||||
func (i ActiveSideState) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
|
||||
case i == 4:
|
||||
return _ActiveSideStateName_1
|
||||
case i == 8:
|
||||
return _ActiveSideStateName_2
|
||||
default:
|
||||
return fmt.Sprintf("ActiveSideState(%d)", i)
|
||||
}
|
||||
}
|
||||
|
||||
var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
|
||||
|
||||
var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
|
||||
_ActiveSideStateName_0[0:21]: 1,
|
||||
_ActiveSideStateName_0[21:42]: 2,
|
||||
_ActiveSideStateName_1[0:23]: 4,
|
||||
_ActiveSideStateName_2[0:14]: 8,
|
||||
}
|
||||
|
||||
// ActiveSideStateString retrieves an enum value from the enum constants string name.
|
||||
// Throws an error if the param is not part of the enum.
|
||||
func ActiveSideStateString(s string) (ActiveSideState, error) {
|
||||
if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
|
||||
return val, nil
|
||||
}
|
||||
return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
|
||||
}
|
||||
|
||||
// ActiveSideStateValues returns all values of the enum
|
||||
func ActiveSideStateValues() []ActiveSideState {
|
||||
return _ActiveSideStateValues
|
||||
}
|
||||
|
||||
// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
|
||||
func (i ActiveSideState) IsAActiveSideState() bool {
|
||||
for _, v := range _ActiveSideStateValues {
|
||||
if i == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
||||
)
|
||||
|
||||
func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
|
||||
js := make([]Job, len(c.Jobs))
|
||||
for i := range c.Jobs {
|
||||
j, err := buildJob(c.Global, c.Jobs[i], parseFlags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if j == nil || j.Name() == "" {
|
||||
panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
|
||||
}
|
||||
js[i] = j
|
||||
}
|
||||
|
||||
return js, nil
|
||||
}
|
||||
|
||||
func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags) (j Job, err error) {
|
||||
cannotBuildJob := func(e error, name string) (Job, error) {
|
||||
return nil, errors.Wrapf(e, "cannot build job %q", name)
|
||||
}
|
||||
// FIXME prettify this
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.SinkJob:
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.SourceJob:
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.SnapJob:
|
||||
j, err = snapJobFromConfig(c, v)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.PushJob:
|
||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.PullJob:
|
||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
default:
|
||||
panic(fmt.Sprintf("implementation error: unknown job type %T", v))
|
||||
}
|
||||
return j, nil
|
||||
|
||||
}
|
||||
|
||||
func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
||||
if len(receivingRootFSs) == 0 {
|
||||
return nil
|
||||
}
|
||||
rfss := make([]string, len(receivingRootFSs))
|
||||
copy(rfss, receivingRootFSs)
|
||||
sort.Slice(rfss, func(i, j int) bool {
|
||||
return strings.Compare(rfss[i], rfss[j]) == -1
|
||||
})
|
||||
// add tailing slash because of hierarchy-simulation
|
||||
// rootfs/ is not root of rootfs2/
|
||||
for i := 0; i < len(rfss); i++ {
|
||||
rfss[i] += "/"
|
||||
}
|
||||
// idea:
|
||||
// no path in rfss must be prefix of another
|
||||
//
|
||||
// rfss is now lexicographically sorted, which means that
|
||||
// if i is prefix of j, i < j (in lexicographical order)
|
||||
// thus,
|
||||
// if any i is prefix of i+n (n >= 1), there is overlap
|
||||
for i := 0; i < len(rfss)-1; i++ {
|
||||
if strings.HasPrefix(rfss[i+1], rfss[i]) {
|
||||
return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
|
||||
if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
|
||||
return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
|
||||
}
|
||||
return bandwidthlimit.Config{
|
||||
Max: int64(in.Max.ToBytes()),
|
||||
BucketCapacity: int64(in.BucketCapacity.ToBytes()),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/util/nodefault"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
)
|
||||
|
||||
type SendingJobConfig interface {
|
||||
GetFilesystems() config.FilesystemsFilter
|
||||
GetSendOptions() *config.SendOptions // must not be nil
|
||||
}
|
||||
|
||||
func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
|
||||
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
}
|
||||
sendOpts := in.GetSendOptions()
|
||||
bwlim, err := buildBandwidthLimitConfig(sendOpts.BandwidthLimit)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build bandwith limit config")
|
||||
}
|
||||
|
||||
sc := &endpoint.SenderConfig{
|
||||
FSF: fsf,
|
||||
JobID: jobID,
|
||||
|
||||
Encrypt: &nodefault.Bool{B: sendOpts.Encrypted},
|
||||
SendRaw: sendOpts.Raw,
|
||||
SendProperties: sendOpts.SendProperties,
|
||||
SendBackupProperties: sendOpts.BackupProperties,
|
||||
SendLargeBlocks: sendOpts.LargeBlocks,
|
||||
SendCompressed: sendOpts.Compressed,
|
||||
SendEmbeddedData: sendOpts.EmbeddedData,
|
||||
SendSaved: sendOpts.Saved,
|
||||
|
||||
BandwidthLimit: bwlim,
|
||||
}
|
||||
|
||||
if err := sc.Validate(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build sender config")
|
||||
}
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
type ReceivingJobConfig interface {
|
||||
GetRootFS() string
|
||||
GetAppendClientIdentity() bool
|
||||
GetRecvOptions() *config.RecvOptions
|
||||
}
|
||||
|
||||
func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
|
||||
rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
|
||||
if err != nil {
|
||||
return rc, errors.New("root_fs is not a valid zfs filesystem path")
|
||||
}
|
||||
if rootFs.Length() <= 0 {
|
||||
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
|
||||
}
|
||||
|
||||
recvOpts := in.GetRecvOptions()
|
||||
|
||||
bwlim, err := buildBandwidthLimitConfig(recvOpts.BandwidthLimit)
|
||||
if err != nil {
|
||||
return rc, errors.Wrap(err, "cannot build bandwith limit config")
|
||||
}
|
||||
|
||||
placeholderEncryption, err := endpoint.PlaceholderCreationEncryptionPropertyString(recvOpts.Placeholder.Encryption)
|
||||
if err != nil {
|
||||
options := []string{}
|
||||
for _, v := range endpoint.PlaceholderCreationEncryptionPropertyValues() {
|
||||
options = append(options, endpoint.PlaceholderCreationEncryptionProperty(v).String())
|
||||
}
|
||||
return rc, errors.Errorf("placeholder encryption value %q is invalid, must be one of %s",
|
||||
recvOpts.Placeholder.Encryption, options)
|
||||
}
|
||||
|
||||
rc = endpoint.ReceiverConfig{
|
||||
JobID: jobID,
|
||||
RootWithoutClientComponent: rootFs,
|
||||
AppendClientIdentity: in.GetAppendClientIdentity(),
|
||||
|
||||
InheritProperties: recvOpts.Properties.Inherit,
|
||||
OverrideProperties: recvOpts.Properties.Override,
|
||||
|
||||
BandwidthLimit: bwlim,
|
||||
|
||||
PlaceholderEncryption: placeholderEncryption,
|
||||
}
|
||||
if err := rc.Validate(); err != nil {
|
||||
return rc, errors.Wrap(err, "cannot build receiver config")
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
)
|
||||
|
||||
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
|
||||
type testCase struct {
|
||||
err bool
|
||||
input []string
|
||||
}
|
||||
tcs := []testCase{
|
||||
{false, nil},
|
||||
{false, []string{}},
|
||||
{false, []string{""}}, // not our job to determine valid paths
|
||||
{false, []string{"a"}},
|
||||
{false, []string{"some/path"}},
|
||||
{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
|
||||
{false, []string{"zroot/foo", "zroot/foobar"}},
|
||||
{true, []string{"zroot/b", "zroot/b"}},
|
||||
{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
|
||||
{false, []string{"a/x", "b/x"}},
|
||||
{false, []string{"a", "b"}},
|
||||
{true, []string{"a", "a"}},
|
||||
{true, []string{"a/x/y", "a/x"}},
|
||||
{true, []string{"a/x", "a/x/y"}},
|
||||
{true, []string{"a/x", "b/x", "a/x/y"}},
|
||||
{true, []string{"a", "a/b", "a/c", "a/b"}},
|
||||
{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Logf("input: %v", tc.input)
|
||||
err := validateReceivingSidesDoNotOverlap(tc.input)
|
||||
if tc.err {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobIDErrorHandling(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: %s
|
||||
type: push
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
filesystems: {"<": true}
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
|
||||
type Case struct {
|
||||
jobName string
|
||||
valid bool
|
||||
}
|
||||
cases := []Case{
|
||||
{"validjobname", true},
|
||||
{"valid with spaces", true},
|
||||
{"invalid\twith\ttabs", false},
|
||||
{"invalid#withdelimiter", false},
|
||||
{"invalid@withdelimiter", false},
|
||||
{"withnewline\\nmiddle", false},
|
||||
{"withnewline\\n", false},
|
||||
{"withslash/", false},
|
||||
{"withslash/inthemiddle", false},
|
||||
{"/", false},
|
||||
}
|
||||
|
||||
for i := range cases {
|
||||
t.Run(cases[i].jobName, func(t *testing.T) {
|
||||
c := cases[i]
|
||||
|
||||
conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
|
||||
require.NoError(t, err, "not expecting yaml-config to know about job ids")
|
||||
require.NotNil(t, conf)
|
||||
jobs, err := JobsFromConfig(conf, config.ParseFlagsNone)
|
||||
|
||||
if c.valid {
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, jobs, 1)
|
||||
assert.Equal(t, c.jobName, jobs[0].Name())
|
||||
} else {
|
||||
t.Logf("error: %s", err)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, jobs)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
|
||||
paths, err := filepath.Glob("../../../internal/config/samples/*")
|
||||
if err != nil {
|
||||
t.Errorf("glob failed: %+v", err)
|
||||
}
|
||||
|
||||
type additionalCheck struct {
|
||||
state int
|
||||
test func(t *testing.T, jobs []Job)
|
||||
}
|
||||
additionalChecks := map[string]*additionalCheck{
|
||||
"bandwidth_limit.yml": {test: testSampleConfig_BandwidthLimit},
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
if path.Ext(p) != ".yml" {
|
||||
t.Logf("skipping file %s", p)
|
||||
continue
|
||||
}
|
||||
|
||||
filename := path.Base(p)
|
||||
t.Logf("checking for presence additonal checks for file %q", filename)
|
||||
additionalCheck := additionalChecks[filename]
|
||||
if additionalCheck == nil {
|
||||
t.Logf("no additional checks")
|
||||
} else {
|
||||
t.Logf("additional check present")
|
||||
additionalCheck.state = 1
|
||||
}
|
||||
|
||||
t.Run(p, func(t *testing.T) {
|
||||
c, err := config.ParseConfig(p)
|
||||
if err != nil {
|
||||
t.Fatalf("error parsing %s:\n%+v", p, err)
|
||||
}
|
||||
|
||||
t.Logf("file: %s", p)
|
||||
t.Log(pretty.Sprint(c))
|
||||
|
||||
jobs, err := JobsFromConfig(c, config.ParseFlagsNoCertCheck)
|
||||
t.Logf("jobs: %#v", jobs)
|
||||
require.NoError(t, err)
|
||||
|
||||
if additionalCheck != nil {
|
||||
additionalCheck.test(t, jobs)
|
||||
additionalCheck.state = 2
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
for basename, c := range additionalChecks {
|
||||
if c.state == 0 {
|
||||
panic("univisited additional check " + basename)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testSampleConfig_BandwidthLimit(t *testing.T, jobs []Job) {
|
||||
require.Len(t, jobs, 3)
|
||||
|
||||
{
|
||||
limitedSink, ok := jobs[0].(*PassiveSide)
|
||||
require.True(t, ok, "%T", jobs[0])
|
||||
limitedSinkMode, ok := limitedSink.mode.(*modeSink)
|
||||
require.True(t, ok, "%T", limitedSink)
|
||||
|
||||
assert.Equal(t, int64(12345), limitedSinkMode.receiverConfig.BandwidthLimit.Max)
|
||||
assert.Equal(t, int64(1<<17), limitedSinkMode.receiverConfig.BandwidthLimit.BucketCapacity)
|
||||
}
|
||||
|
||||
{
|
||||
limitedPush, ok := jobs[1].(*ActiveSide)
|
||||
require.True(t, ok, "%T", jobs[1])
|
||||
limitedPushMode, ok := limitedPush.mode.(*modePush)
|
||||
require.True(t, ok, "%T", limitedPush)
|
||||
|
||||
assert.Equal(t, int64(54321), limitedPushMode.senderConfig.BandwidthLimit.Max)
|
||||
assert.Equal(t, int64(1024), limitedPushMode.senderConfig.BandwidthLimit.BucketCapacity)
|
||||
}
|
||||
|
||||
{
|
||||
unlimitedSink, ok := jobs[2].(*PassiveSide)
|
||||
require.True(t, ok, "%T", jobs[2])
|
||||
unlimitedSinkMode, ok := unlimitedSink.mode.(*modeSink)
|
||||
require.True(t, ok, "%T", unlimitedSink)
|
||||
|
||||
max := unlimitedSinkMode.receiverConfig.BandwidthLimit.Max
|
||||
assert.Less(t, max, int64(0), max, "unlimited mode <=> negative value for .Max, see bandwidthlimit.Config")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestReplicationOptions(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: push
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
filesystems: {"<": true}
|
||||
%s
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
|
||||
type Test struct {
|
||||
name string
|
||||
input string
|
||||
expectOk func(t *testing.T, a *ActiveSide, m *modePush)
|
||||
expectError bool
|
||||
}
|
||||
|
||||
tests := []Test{
|
||||
{
|
||||
name: "defaults",
|
||||
input: `
|
||||
replication: {}
|
||||
`,
|
||||
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {},
|
||||
},
|
||||
{
|
||||
name: "steps_zero",
|
||||
input: `
|
||||
replication:
|
||||
concurrency:
|
||||
steps: 0
|
||||
`,
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "size_estimates_zero",
|
||||
input: `
|
||||
replication:
|
||||
concurrency:
|
||||
size_estimates: 0
|
||||
`,
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "custom_values",
|
||||
input: `
|
||||
replication:
|
||||
concurrency:
|
||||
steps: 23
|
||||
size_estimates: 42
|
||||
`,
|
||||
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {
|
||||
assert.Equal(t, 23, a.replicationDriverConfig.StepQueueConcurrency)
|
||||
assert.Equal(t, 42, m.plannerPolicy.SizeEstimationConcurrency)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "negative_values_forbidden",
|
||||
input: `
|
||||
replication:
|
||||
concurrency:
|
||||
steps: -23
|
||||
size_estimates: -42
|
||||
`,
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
|
||||
for _, ts := range tests {
|
||||
t.Run(ts.name, func(t *testing.T) {
|
||||
assert.True(t, (ts.expectError) != (ts.expectOk != nil))
|
||||
|
||||
cstr := fill(ts.input)
|
||||
t.Logf("testing config:\n%s", cstr)
|
||||
c, err := config.ParseConfigBytes([]byte(cstr))
|
||||
require.NoError(t, err)
|
||||
jobs, err := JobsFromConfig(c, config.ParseFlagsNone)
|
||||
if ts.expectOk != nil {
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, c)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, jobs, 1)
|
||||
a := jobs[0].(*ActiveSide)
|
||||
m := a.mode.(*modePush)
|
||||
ts.expectOk(t, a, m)
|
||||
} else if ts.expectError {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
t.Fatalf("test must define expectOk or expectError")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
func GetLogger(ctx context.Context) Logger {
|
||||
return logging.GetLogger(ctx, logging.SubsysJob)
|
||||
}
|
||||
|
||||
type Job interface {
|
||||
Name() string
|
||||
Run(ctx context.Context)
|
||||
Status() *Status
|
||||
RegisterMetrics(registerer prometheus.Registerer)
|
||||
// Jobs that return a subtree of the dataset hierarchy
|
||||
// must return the root of that subtree as rfs and ok = true
|
||||
OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
|
||||
SenderConfig() *endpoint.SenderConfig
|
||||
}
|
||||
|
||||
type Type string
|
||||
|
||||
const (
|
||||
TypeInternal Type = "internal"
|
||||
TypeSnap Type = "snap"
|
||||
TypePush Type = "push"
|
||||
TypeSink Type = "sink"
|
||||
TypePull Type = "pull"
|
||||
TypeSource Type = "source"
|
||||
)
|
||||
|
||||
type Status struct {
|
||||
Type Type
|
||||
JobSpecific interface{}
|
||||
}
|
||||
|
||||
func (s *Status) MarshalJSON() ([]byte, error) {
|
||||
typeJson, err := json.Marshal(s.Type)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobJSON, err := json.Marshal(s.JobSpecific)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := map[string]json.RawMessage{
|
||||
"type": typeJson,
|
||||
string(s.Type): jobJSON,
|
||||
}
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
func (s *Status) UnmarshalJSON(in []byte) (err error) {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(in, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
tJSON, ok := m["type"]
|
||||
if !ok {
|
||||
return fmt.Errorf("field 'type' not found")
|
||||
}
|
||||
if err := json.Unmarshal(tJSON, &s.Type); err != nil {
|
||||
return err
|
||||
}
|
||||
key := string(s.Type)
|
||||
jobJSON, ok := m[key]
|
||||
if !ok {
|
||||
return fmt.Errorf("field '%s', not found", key)
|
||||
}
|
||||
switch s.Type {
|
||||
case TypeSnap:
|
||||
var st SnapJobStatus
|
||||
err = json.Unmarshal(jobJSON, &st)
|
||||
s.JobSpecific = &st
|
||||
|
||||
case TypePull:
|
||||
fallthrough
|
||||
case TypePush:
|
||||
var st ActiveSideStatus
|
||||
err = json.Unmarshal(jobJSON, &st)
|
||||
s.JobSpecific = &st
|
||||
|
||||
case TypeSource:
|
||||
fallthrough
|
||||
case TypeSink:
|
||||
var st PassiveStatus
|
||||
err = json.Unmarshal(jobJSON, &st)
|
||||
s.JobSpecific = &st
|
||||
|
||||
case TypeInternal:
|
||||
// internal jobs do not report specifics
|
||||
default:
|
||||
err = fmt.Errorf("unknown job type '%s'", key)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/rpc"
|
||||
"github.com/zrepl/zrepl/internal/transport"
|
||||
"github.com/zrepl/zrepl/internal/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
)
|
||||
|
||||
type PassiveSide struct {
|
||||
mode passiveMode
|
||||
name endpoint.JobID
|
||||
listen transport.AuthenticatedListenerFactory
|
||||
}
|
||||
|
||||
type passiveMode interface {
|
||||
Handler() rpc.Handler
|
||||
RunPeriodic(ctx context.Context)
|
||||
SnapperReport() *snapper.Report // may be nil
|
||||
Type() Type
|
||||
}
|
||||
|
||||
type modeSink struct {
|
||||
receiverConfig endpoint.ReceiverConfig
|
||||
}
|
||||
|
||||
func (m *modeSink) Type() Type { return TypeSink }
|
||||
|
||||
func (m *modeSink) Handler() rpc.Handler {
|
||||
return endpoint.NewReceiver(m.receiverConfig)
|
||||
}
|
||||
|
||||
func (m *modeSink) RunPeriodic(_ context.Context) {}
|
||||
func (m *modeSink) SnapperReport() *snapper.Report { return nil }
|
||||
|
||||
func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
|
||||
m = &modeSink{}
|
||||
|
||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type modeSource struct {
|
||||
senderConfig *endpoint.SenderConfig
|
||||
snapper snapper.Snapper
|
||||
}
|
||||
|
||||
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
|
||||
// FIXME exact dedup of modePush
|
||||
m = &modeSource{}
|
||||
|
||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "send options")
|
||||
}
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *modeSource) Type() Type { return TypeSource }
|
||||
|
||||
func (m *modeSource) Handler() rpc.Handler {
|
||||
return endpoint.NewSender(*m.senderConfig)
|
||||
}
|
||||
|
||||
func (m *modeSource) RunPeriodic(ctx context.Context) {
|
||||
m.snapper.Run(ctx, nil)
|
||||
}
|
||||
|
||||
func (m *modeSource) SnapperReport() *snapper.Report {
|
||||
r := m.snapper.Report()
|
||||
return &r
|
||||
}
|
||||
|
||||
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}, parseFlags config.ParseFlags) (s *PassiveSide, err error) {
|
||||
|
||||
s = &PassiveSide{}
|
||||
|
||||
s.name, err = endpoint.MakeJobID(in.Name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "invalid job name")
|
||||
}
|
||||
|
||||
switch v := configJob.(type) {
|
||||
case *config.SinkJob:
|
||||
s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
|
||||
case *config.SourceJob:
|
||||
s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err // no wrapping necessary
|
||||
}
|
||||
|
||||
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve, parseFlags); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build listener factory")
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (j *PassiveSide) Name() string { return j.name.String() }
|
||||
|
||||
type PassiveStatus struct {
|
||||
Snapper *snapper.Report
|
||||
}
|
||||
|
||||
func (s *PassiveSide) Status() *Status {
|
||||
st := &PassiveStatus{
|
||||
Snapper: s.mode.SnapperReport(),
|
||||
}
|
||||
return &Status{Type: s.mode.Type(), JobSpecific: st}
|
||||
}
|
||||
|
||||
func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
sink, ok := j.mode.(*modeSink)
|
||||
if !ok {
|
||||
_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
|
||||
return nil, false
|
||||
}
|
||||
return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
|
||||
}
|
||||
|
||||
func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
|
||||
source, ok := j.mode.(*modeSource)
|
||||
if !ok {
|
||||
_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
|
||||
return nil
|
||||
}
|
||||
return source.senderConfig
|
||||
}
|
||||
|
||||
func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
|
||||
|
||||
func (j *PassiveSide) Run(ctx context.Context) {
|
||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
|
||||
defer endTask()
|
||||
log := GetLogger(ctx)
|
||||
defer log.Info("job exiting")
|
||||
{
|
||||
ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
|
||||
defer endTask()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go j.mode.RunPeriodic(ctx)
|
||||
}
|
||||
|
||||
handler := j.mode.Handler()
|
||||
if handler == nil {
|
||||
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
|
||||
}
|
||||
|
||||
ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
|
||||
// the handlerCtx is clean => need to inherit logging and tracing config from job context
|
||||
handlerCtx = logging.WithInherit(handlerCtx, ctx)
|
||||
handlerCtx = trace.WithInherit(handlerCtx, ctx)
|
||||
|
||||
handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
|
||||
defer endTask()
|
||||
handler(handlerCtx)
|
||||
}
|
||||
|
||||
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
|
||||
server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
|
||||
|
||||
listener, err := j.listen()
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot listen")
|
||||
return
|
||||
}
|
||||
|
||||
server.Serve(ctx, listener)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package reset
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
type contextKey int
|
||||
|
||||
const contextKeyReset contextKey = iota
|
||||
|
||||
func Wait(ctx context.Context) <-chan struct{} {
|
||||
wc, ok := ctx.Value(contextKeyReset).(chan struct{})
|
||||
if !ok {
|
||||
wc = make(chan struct{})
|
||||
}
|
||||
return wc
|
||||
}
|
||||
|
||||
type Func func() error
|
||||
|
||||
var AlreadyReset = errors.New("already reset")
|
||||
|
||||
func Context(ctx context.Context) (context.Context, Func) {
|
||||
wc := make(chan struct{})
|
||||
wuf := func() error {
|
||||
select {
|
||||
case wc <- struct{}{}:
|
||||
return nil
|
||||
default:
|
||||
return AlreadyReset
|
||||
}
|
||||
}
|
||||
return context.WithValue(ctx, contextKeyReset, wc), wuf
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
||||
"github.com/zrepl/zrepl/internal/util/nodefault"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
)
|
||||
|
||||
type SnapJob struct {
|
||||
name endpoint.JobID
|
||||
fsfilter zfs.DatasetFilter
|
||||
snapper snapper.Snapper
|
||||
|
||||
prunerFactory *pruner.LocalPrunerFactory
|
||||
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
|
||||
prunerMtx sync.Mutex
|
||||
pruner *pruner.Pruner
|
||||
}
|
||||
|
||||
func (j *SnapJob) Name() string { return j.name.String() }
|
||||
|
||||
func (j *SnapJob) Type() Type { return TypeSnap }
|
||||
|
||||
func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
|
||||
j = &SnapJob{}
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
}
|
||||
j.fsfilter = fsf
|
||||
|
||||
if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
j.name, err = endpoint.MakeJobID(in.Name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "invalid job name")
|
||||
}
|
||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "pruning",
|
||||
Name: "time",
|
||||
Help: "seconds spent in pruner",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
}, []string{"prune_side"})
|
||||
j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
registerer.MustRegister(j.promPruneSecs)
|
||||
}
|
||||
|
||||
type SnapJobStatus struct {
|
||||
Pruning *pruner.Report
|
||||
Snapshotting *snapper.Report // may be nil
|
||||
}
|
||||
|
||||
func (j *SnapJob) Status() *Status {
|
||||
s := &SnapJobStatus{}
|
||||
t := j.Type()
|
||||
j.prunerMtx.Lock()
|
||||
if j.pruner != nil {
|
||||
s.Pruning = j.pruner.Report()
|
||||
}
|
||||
j.prunerMtx.Unlock()
|
||||
r := j.snapper.Report()
|
||||
s.Snapshotting = &r
|
||||
return &Status{Type: t, JobSpecific: s}
|
||||
}
|
||||
|
||||
func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
||||
|
||||
func (j *SnapJob) Run(ctx context.Context) {
|
||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
|
||||
defer endTask()
|
||||
log := GetLogger(ctx)
|
||||
|
||||
defer log.Info("job exiting")
|
||||
|
||||
periodicDone := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
|
||||
defer endTask()
|
||||
go j.snapper.Run(periodicCtx, periodicDone)
|
||||
|
||||
invocationCount := 0
|
||||
outer:
|
||||
for {
|
||||
log.Info("wait for wakeups")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
|
||||
case <-wakeup.Wait(ctx):
|
||||
case <-periodicDone:
|
||||
}
|
||||
invocationCount++
|
||||
|
||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
||||
j.doPrune(invocationCtx)
|
||||
endSpan()
|
||||
}
|
||||
}
|
||||
|
||||
// Adaptor that implements pruner.History around a pruner.Target.
|
||||
// The ReplicationCursor method is Get-op only and always returns
|
||||
// the filesystem's most recent version's GUID.
|
||||
//
|
||||
// TODO:
|
||||
// This is a work-around for the current package daemon/pruner
|
||||
// and package pruning.Snapshot limitation: they require the
|
||||
// `Replicated` getter method be present, but obviously,
|
||||
// a local job like SnapJob can't deliver on that.
|
||||
// But the pruner.Pruner gives up on an FS if no replication
|
||||
// cursor is present, which is why this pruner returns the
|
||||
// most recent filesystem version.
|
||||
type alwaysUpToDateReplicationCursorHistory struct {
|
||||
// the Target passed as Target to BuildLocalPruner
|
||||
target pruner.Target
|
||||
}
|
||||
|
||||
var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
||||
|
||||
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
|
||||
fsvReq := &pdu.ListFilesystemVersionsReq{
|
||||
Filesystem: req.GetFilesystem(),
|
||||
}
|
||||
res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fsvs := res.GetVersions()
|
||||
if len(fsvs) <= 0 {
|
||||
return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
|
||||
}
|
||||
// always return must recent version
|
||||
sort.Slice(fsvs, func(i, j int) bool {
|
||||
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
|
||||
})
|
||||
mostRecent := fsvs[len(fsvs)-1]
|
||||
return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
|
||||
}
|
||||
|
||||
func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
|
||||
return h.target.ListFilesystems(ctx, req)
|
||||
}
|
||||
|
||||
func (j *SnapJob) doPrune(ctx context.Context) {
|
||||
ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
|
||||
defer endSpan()
|
||||
log := GetLogger(ctx)
|
||||
sender := endpoint.NewSender(endpoint.SenderConfig{
|
||||
JobID: j.name,
|
||||
FSF: j.fsfilter,
|
||||
// FIXME the following config fields are irrelevant for SnapJob
|
||||
// because the endpoint is only used as pruner.Target.
|
||||
// However, the implementation requires them to be set.
|
||||
Encrypt: &nodefault.Bool{B: true},
|
||||
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
|
||||
})
|
||||
j.prunerMtx.Lock()
|
||||
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
||||
j.prunerMtx.Unlock()
|
||||
log.Info("start pruning")
|
||||
j.pruner.Prune()
|
||||
log.Info("finished pruning")
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package wakeup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
type contextKey int
|
||||
|
||||
const contextKeyWakeup contextKey = iota
|
||||
|
||||
func Wait(ctx context.Context) <-chan struct{} {
|
||||
wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
|
||||
if !ok {
|
||||
wc = make(chan struct{})
|
||||
}
|
||||
return wc
|
||||
}
|
||||
|
||||
type Func func() error
|
||||
|
||||
var AlreadyWokenUp = errors.New("already woken up")
|
||||
|
||||
func Context(ctx context.Context) (context.Context, Func) {
|
||||
wc := make(chan struct{})
|
||||
wuf := func() error {
|
||||
select {
|
||||
case wc <- struct{}{}:
|
||||
return nil
|
||||
default:
|
||||
return AlreadyWokenUp
|
||||
}
|
||||
}
|
||||
return context.WithValue(ctx, contextKeyWakeup, wc), wuf
|
||||
}
|
||||
Reference in New Issue
Block a user