WIP runtime-controllable concurrency for replication

Changes done so far:
- signal to route concurrency request up to the stepQueue
    - pretty hacky, no status reporting yet
- stepQueue upsizing (confirmed that it works, no intermediary commit)

Stuck at: stepQueue downsizing
- idea was to have the stepQueue signal to the activated step that it
  should suspend
- ideally, we'd just kill everything associated with the step and track
  it as restartable
    - the driver model doesn't allow for that though within an attempt
    - would need to start a separate run for the step
- less perfect: tell the downsized steps to stop copying, but leave
  all zfs sends + rpc conns open
    - - doesn't give back the resoures that a step aquired before
      being selected as downsize victim (open zfs processe + TCP conn,
      some memory in the pipe)
    - - looks weird to user if the ps aux
    - + re-waking a step is easy: just tell it to proceed with copying
    - (the impl would likely pass a check function down into Step.do
      and have it check that functino periodically. the suspend should
      be acknowledged, and stepQueue should only remove the step from
      the active queue _after_ that step has acknowledged that is
      suspended)
This commit is contained in:
Christian Schwarz
2020-02-01 15:28:52 +01:00
parent b8d9f4ba92
commit 17add553d3
14 changed files with 244 additions and 75 deletions
+12
View File
@@ -8,6 +8,7 @@ import (
"io"
"net"
"net/http"
"strconv"
"time"
"github.com/pkg/errors"
@@ -47,6 +48,8 @@ func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { r
func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
func (j *controlJob) SetConcurrency(concurrency int) error { return errors.Errorf("not supported") }
var promControl struct {
requestBegin *prometheus.CounterVec
requestFinished *prometheus.HistogramVec
@@ -126,6 +129,7 @@ func (j *controlJob) Run(ctx context.Context) {
type reqT struct {
Name string
Op string
Data string
}
var req reqT
if decoder(&req) != nil {
@@ -138,6 +142,14 @@ func (j *controlJob) Run(ctx context.Context) {
err = j.jobs.wakeup(req.Name)
case "reset":
err = j.jobs.reset(req.Name)
case "set-concurrency":
var concurrency int
concurrency, err = strconv.Atoi(req.Data) // shadow
if err != nil {
// fallthrough outer
} else {
err = j.jobs.setConcurrency(req.Name, concurrency)
}
default:
err = fmt.Errorf("operation %q is invalid", req.Op)
}