adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: reset handling, unified wait token TODO: task-level invocations (replication, snapshotting, pruning)

Browse Source
This commit is contained in:
Christian Schwarz
2021-03-21 23:47:37 +01:00
parent 97a14dba90
commit a4a2cb2833
9 changed files with 253 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/control.go
+57 -10
View File
@@ -73,14 +73,25 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
}
const (
ControlJobEndpointPProf string = "/debug/pprof"
ControlJobEndpointVersion string = "/version"
ControlJobEndpointStatus string = "/status"
ControlJobEndpointSignalActive string = "/signal/active"
ControlJobEndpointPollActive string = "/poll/active"
ControlJobEndpointPProf string = "/debug/pprof"
ControlJobEndpointVersion string = "/version"
ControlJobEndpointStatus string = "/status"
ControlJobEndpointTriggerActive string = "/signal/active"
ControlJobEndpointPollActive string = "/poll/active"
ControlJobEndpointResetActive string = "/reset/active"
)
type ControlJobEndpointSignalActiveRequest struct {
type ControlJobEndpointTriggerActiveRequest struct {
Job string
job.ActiveSideTriggerRequest
}
type ControlJobEndpointResetActiveRequest struct {
Job string
job.ActiveSideResetRequest
}
type ControlJobEndpointWaitActiveRequest struct {
Job string
job.ActiveSidePollRequest
}
@@ -137,7 +148,7 @@ func (j *controlJob) Run(ctx context.Context) {
}})
mux.Handle(ControlJobEndpointPollActive, requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
var req ControlJobEndpointSignalActiveRequest
var req ControlJobEndpointWaitActiveRequest
if decoder(&req) != nil {
return nil, errors.Errorf("decode failed")
}
@@ -164,11 +175,11 @@ func (j *controlJob) Run(ctx context.Context) {
return res, err
}}})
mux.Handle(ControlJobEndpointSignalActive,
mux.Handle(ControlJobEndpointTriggerActive,
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
type reqT struct {
Job string
job.ActiveSideSignalRequest
job.ActiveSideTriggerRequest
}
var req reqT
if decoder(&req) != nil {
@@ -192,7 +203,43 @@ func (j *controlJob) Run(ctx context.Context) {
return v, err
}
res, err := ajo.Signal(req.ActiveSideSignalRequest)
res, err := ajo.Trigger(req.ActiveSideTriggerRequest)
j.jobs.m.RUnlock()
return res, err
}}})
mux.Handle(ControlJobEndpointResetActive,
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
type reqT struct {
Job string
job.ActiveSideResetRequest
}
var req reqT
if decoder(&req) != nil {
return nil, errors.Errorf("decode failed")
}
// FIXME dedup the following code with ControlJobEndpointPollActive
j.jobs.m.RLock()
jo, ok := j.jobs.jobs[req.Job]
if !ok {
j.jobs.m.RUnlock()
return struct{}{}, fmt.Errorf("unknown job name %q", req.Job)
}
ajo, ok := jo.(*job.ActiveSide)
if !ok {
v, err = struct{}{}, fmt.Errorf("job %q is not an active side (it's a %T)", jo.Name(), jo)
j.jobs.m.RUnlock()
return v, err
}
res, err := ajo.Reset(req.ActiveSideResetRequest)
j.jobs.m.RUnlock()
daemon/job/active.go
+78 -33
View File
@@ -8,13 +8,11 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/envconst"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/pruner"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -46,7 +44,8 @@ type ActiveSide struct {
tasks activeSideTasks
nextInvocationId uint64
activeInvocationId uint64 // 0 <=> inactive
signal chan struct{}
trigger chan struct{}
reset chan uint64
}
//go:generate enumer -type=ActiveSideState
@@ -437,7 +436,8 @@ func (j *ActiveSide) Run(ctx context.Context) {
wakePeriodic := make(chan struct{})
go j.mode.RunPeriodic(periodicCtx, wakePeriodic, periodicDone)
j.signal = make(chan struct{})
j.trigger = make(chan struct{})
j.reset = make(chan uint64)
j.nextInvocationId = 1
outer:
@@ -448,7 +448,7 @@ outer:
log.WithError(ctx.Err()).Info("context")
break outer
case <-j.signal:
case <-j.trigger:
j.mode.ResetConnectBackoff()
case <-periodicDone:
}
@@ -456,10 +456,38 @@ outer:
j.tasksMtx.Lock()
j.activeInvocationId = j.nextInvocationId
j.nextInvocationId++
thisInvocation := j.activeInvocationId // stack-local, for use in reset-handler goroutine below
j.tasksMtx.Unlock()
// setup the invocation context
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", j.nextInvocationId))
invocationCtx, cancelInvocation := context.WithCancel(invocationCtx)
// setup the goroutine that waits for task resets
// Task resets are converted into cancellations of the invocation context.
waitForResetCtx, stopWaitForReset := context.WithCancel(ctx)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
select {
case <-waitForResetCtx.Done():
return
case reqResetInvocation := <-j.reset:
l := log.WithField("requested_invocation_id", reqResetInvocation).
WithField("this_invocation_id", thisInvocation)
if reqResetInvocation == thisInvocation {
l.Info("reset received, cancelling current invocation")
cancelInvocation()
} else {
l.Debug("received reset for invocation id that is not us, discarding request")
}
}
}()
j.do(invocationCtx)
stopWaitForReset()
wg.Wait()
j.tasksMtx.Lock()
j.activeInvocationId = 0
@@ -471,7 +499,6 @@ outer:
type ActiveSidePollRequest struct {
InvocationId uint64
What string
}
type ActiveSidePollResponse struct {
@@ -493,22 +520,52 @@ func (j *ActiveSide) Poll(req ActiveSidePollRequest) (*ActiveSidePollResponse, e
}
}
switch req.What {
case "invocation":
var done bool
if j.activeInvocationId == 0 {
done = waitForId < j.nextInvocationId
} else {
done = waitForId < j.activeInvocationId
}
res := &ActiveSidePollResponse{Done: done, InvocationId: waitForId}
return res, nil
default:
return nil, fmt.Errorf("unknown wait target %q", req.What)
var done bool
if j.activeInvocationId == 0 {
done = waitForId < j.nextInvocationId
} else {
done = waitForId < j.activeInvocationId
}
res := &ActiveSidePollResponse{Done: done, InvocationId: waitForId}
return res, nil
}
type ActiveSideSignalRequest struct {
type ActiveSideResetRequest struct {
InvocationId uint64
}
type ActiveSideResetResponse struct {
InvocationId uint64
}
func (j *ActiveSide) Reset(req ActiveSideResetRequest) (*ActiveSideResetResponse, error) {
j.tasksMtx.Lock()
defer j.tasksMtx.Unlock()
resetId := req.InvocationId
if req.InvocationId == 0 {
// handle the case where the client doesn't know what the current invocation id is
resetId = j.activeInvocationId
}
if resetId == 0 {
return nil, fmt.Errorf("no active invocation")
}
if resetId != j.activeInvocationId {
return nil, fmt.Errorf("active invocation (%d) is not the invocation requested for reset (%d); (active invocation '0' indicates no active invocation)", j.activeInvocationId, resetId)
}
// non-blocking send (.Run() must not hold mutex while waiting for resets)
select {
case j.reset <- resetId:
default:
}
return &ActiveSideResetResponse{InvocationId: resetId}, nil
}
type ActiveSideTriggerRequest struct {
What string
}
@@ -516,7 +573,7 @@ type ActiveSideSignalResponse struct {
InvocationId uint64
}
func (j *ActiveSide) Signal(req ActiveSideSignalRequest) (*ActiveSideSignalResponse, error) {
func (j *ActiveSide) Trigger(req ActiveSideTriggerRequest) (*ActiveSideSignalResponse, error) {
// switch req.What {
// case "replication":
// invocationId, err = j.jobs.doreplication(req.Name)
@@ -539,7 +596,7 @@ func (j *ActiveSide) Signal(req ActiveSideSignalRequest) (*ActiveSideSignalRespo
}
// non-blocking send (.Run() must not hold mutex while waiting for signals)
select {
case j.signal <- struct{}{}:
case j.trigger <- struct{}{}:
default:
}
j.tasksMtx.Unlock()
@@ -554,18 +611,6 @@ func (j *ActiveSide) do(ctx context.Context) {
j.mode.ConnectEndpoints(ctx, j.connecter)
defer j.mode.DisconnectEndpoints()
// allow cancellation of an invocation (this function)
ctx, cancelThisRun := context.WithCancel(ctx)
defer cancelThisRun()
go func() {
select {
case <-reset.Wait(ctx):
log.Info("reset received, cancelling current invocation")
cancelThisRun()
case <-ctx.Done():
}
}()
sender, receiver := j.mode.SenderReceiver()
{
daemon/job/snapjob.go
+1 -2
View File
@@ -9,7 +9,6 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/job/doreplication"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/util/nodefault"
@@ -118,7 +117,7 @@ outer:
log.WithError(ctx.Err()).Info("context")
break outer
case <-doreplication.Wait(ctx):
// case <-doreplication.Wait(ctx):
case <-periodicDone:
}
invocationCount++
daemon/snapper/snapper.go
+8 -9
View File
@@ -9,7 +9,6 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/daemon/job/dosnapshot"
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
@@ -211,10 +210,10 @@ func syncUp(a args, u updater) state {
return u(func(s *Snapper) {
s.state = Planning
}).sf()
case <-dosnapshot.Wait(a.ctx):
return u(func(s *Snapper) {
s.state = Planning
}).sf()
// case <-dosnapshot.Wait(a.ctx):
// return u(func(s *Snapper) {
// s.state = Planning
// }).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}
@@ -383,10 +382,10 @@ func wait(a args, u updater) state {
return u(func(snapper *Snapper) {
snapper.state = Planning
}).sf()
case <-dosnapshot.Wait(a.ctx):
return u(func(snapper *Snapper) {
snapper.state = Planning
}).sf()
// case <-dosnapshot.Wait(a.ctx):
// return u(func(snapper *Snapper) {
// snapper.state = Planning
// }).sf()
case <-a.ctx.Done():
return onMainCtxDone(a.ctx, u)
}