adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replication + endpoint: replication guarantees: guarantee_{resumability,incremental,nothing}

Browse Source 
This commit

- adds a configuration in which no step holds, replication cursors, etc. are created
- removes the send.step_holds.disable_incremental setting
- creates a new config option `replication` for active-side jobs
- adds the replication.protection.{initial,incremental} settings, each
  of which can have values
    - `guarantee_resumability`
    - `guarantee_incremental`
    - `guarantee_nothing`
  (refer to docs/configuration/replication.rst for semantics)

The `replication` config from an active side is sent to both endpoint.Sender and endpoint.Receiver
for each replication step. Sender and Receiver then act accordingly.

For `guarantee_incremental`, we add the new `tentative-replication-cursor` abstraction.
The necessity for that abstraction is outlined in https://github.com/zrepl/zrepl/issues/340.

fixes https://github.com/zrepl/zrepl/issues/340
This commit is contained in:
Christian Schwarz
2020-06-27 23:53:33 +02:00
parent 27673a23e9
commit 30cdc1430e
36 changed files with 1503 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/job/active.go
+18 -26
View File
@@ -12,7 +12,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/pruner"
@@ -145,23 +144,24 @@ func (m *modePush) ResetConnectBackoff() {
func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
m := &modePush{}
var err error
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
m.senderConfig, err = buildSenderConfig(in, jobID)
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
return nil, errors.Wrap(err, "sender config")
}
m.senderConfig = &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
JobID: jobID,
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.Wrap(err, "field `replication`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
ReplicationConfig: *replicationConfig,
}
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
return nil, errors.Wrap(err, "cannot build snapper")
}
@@ -173,7 +173,6 @@ type modePull struct {
receiver *endpoint.Receiver
receiverConfig endpoint.ReceiverConfig
sender *rpc.Client
rootFS *zfs.DatasetPath
plannerPolicy *logic.PlannerPolicy
interval config.PositiveDurationOrManual
}
@@ -247,26 +246,19 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
m = &modePull{}
m.interval = in.Interval
m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.New("RootFS is not a valid zfs filesystem path")
}
if m.rootFS.Length() <= 0 {
return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
return nil, errors.Wrap(err, "field `replication`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.DontCare,
EncryptedSend: logic.DontCare,
ReplicationConfig: *replicationConfig,
}
m.receiverConfig = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: m.rootFS,
AppendClientIdentity: false, // !
UpdateLastReceivedHold: true,
}
if err := m.receiverConfig.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build receiver config")
m.receiverConfig, err = buildReceiverConfig(in, jobID)
if err != nil {
return nil, err
}
return m, nil
@@ -365,7 +357,7 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
return nil, false
}
return pull.rootFS.Copy(), true
return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
}
func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
daemon/job/build_jobs_sendrecvoptions.go
+55
View File
@@ -0,0 +1,55 @@
package job
import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/zfs"
)
type SendingJobConfig interface {
GetFilesystems() config.FilesystemsFilter
GetSendOptions() *config.SendOptions // must not be nil
}
func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
return &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.GetSendOptions().Encrypted},
JobID: jobID,
}, nil
}
type ReceivingJobConfig interface {
GetRootFS() string
GetAppendClientIdentity() bool
GetRecvOptions() *config.RecvOptions
}
func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
if err != nil {
return rc, errors.New("root_fs is not a valid zfs filesystem path")
}
if rootFs.Length() <= 0 {
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
}
rc = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootFs,
AppendClientIdentity: in.GetAppendClientIdentity(),
}
if err := rc.Validate(); err != nil {
return rc, errors.Wrap(err, "cannot build receiver config")
}
return rc, nil
}
daemon/job/passive.go
+6 -22
View File
@@ -9,7 +9,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -48,19 +47,9 @@ func (m *modeSink) SnapperReport() *snapper.Report { return nil }
func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
m = &modeSink{}
rootDataset, err := zfs.NewDatasetPath(in.RootFS)
m.receiverConfig, err = buildReceiverConfig(in, jobID)
if err != nil {
return nil, errors.New("root dataset is not a valid zfs filesystem path")
}
m.receiverConfig = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootDataset,
AppendClientIdentity: true, // !
UpdateLastReceivedHold: true,
}
if err := m.receiverConfig.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build receiver config")
return nil, err
}
return m, nil
@@ -74,18 +63,13 @@ type modeSource struct {
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
// FIXME exact dedup of modePush
m = &modeSource{}
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
m.senderConfig, err = buildSenderConfig(in, jobID)
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
m.senderConfig = &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
JobID: jobID,
return nil, errors.Wrap(err, "send options")
}
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
return nil, errors.Wrap(err, "cannot build snapper")
}
daemon/job/snapjob.go
-2
View File
@@ -175,8 +175,6 @@ func (j *SnapJob) doPrune(ctx context.Context) {
FSF: j.fsfilter,
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
Encrypt: &zfs.NilBool{B: true},
// FIXME DisableIncrementalStepHolds setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
DisableIncrementalStepHolds: false,
})
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
log.Info("start pruning")