WIP endpoint abstractions + pruning integration / pruner rewrite
This commit is contained in:
Christian Schwarz
@@ -0,0 +1,454 @@
|
||||
package pruner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/pruning"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type Pruner struct {
|
||||
fsfilter endpoint.FSFilter
|
||||
jid endpoint.JobID
|
||||
side Side
|
||||
keepRules []pruning.KeepRule
|
||||
|
||||
// all channels consumed by the run loop
|
||||
reportReqs chan reportRequest
|
||||
stopReqs chan stopRequest
|
||||
done chan struct{}
|
||||
fsListRes chan fsListRes
|
||||
|
||||
state State
|
||||
|
||||
listFilesystemsError error // only in state StateListFilesystemsError
|
||||
fsPruners []*FSPruner // only in state StateFanOutFilesystems
|
||||
}
|
||||
|
||||
//go:generate enumer -type=State -json
|
||||
type State int
|
||||
|
||||
const (
|
||||
StateInitialized State = iota
|
||||
StateListFilesystems
|
||||
StateListFilesystemsError
|
||||
StateFanOutFilesystems
|
||||
StateDone
|
||||
)
|
||||
|
||||
type Report struct {
|
||||
State State
|
||||
ListFilesystemsError error // only valid in StateListFilesystemsError
|
||||
Filesystems []*FSReport // valid from StateFanOutFilesystems
|
||||
}
|
||||
|
||||
type reportRequest struct {
|
||||
ctx context.Context
|
||||
reply chan *Report
|
||||
}
|
||||
|
||||
type runRequest struct {
|
||||
complete chan struct{}
|
||||
}
|
||||
|
||||
type stopRequest struct {
|
||||
complete chan struct{}
|
||||
}
|
||||
|
||||
type fsListRes struct {
|
||||
filesystems []*zfs.DatasetPath
|
||||
err error
|
||||
}
|
||||
|
||||
type Side interface {
|
||||
// may return both nil, indicating there is no replication position
|
||||
GetReplicationPosition(ctx context.Context, fs string) (*zfs.FilesystemVersion, error)
|
||||
isSide() Side
|
||||
}
|
||||
|
||||
func NewPruner(fsfilter endpoint.FSFilter, jid endpoint.JobID, side Side, keepRules []pruning.KeepRule) *Pruner {
|
||||
return &Pruner{
|
||||
fsfilter,
|
||||
jid,
|
||||
side,
|
||||
keepRules,
|
||||
make(chan reportRequest),
|
||||
make(chan stopRequest),
|
||||
make(chan struct{}),
|
||||
make(chan fsListRes),
|
||||
StateInitialized,
|
||||
nil,
|
||||
nil,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pruner) Run(ctx context.Context) *Report {
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
if p.state != StateInitialized {
|
||||
panic("Run can onl[y be called once")
|
||||
}
|
||||
|
||||
go func() {
|
||||
fss, err := zfs.ZFSListMapping(ctx, p.fsfilter)
|
||||
p.fsListRes <- fsListRes{fss, err}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case res := <-p.fsListRes:
|
||||
if res.err != nil {
|
||||
p.state = StateListFilesystemsError
|
||||
p.listFilesystemsError = res.err
|
||||
close(p.done)
|
||||
continue
|
||||
}
|
||||
|
||||
p.state = StateFanOutFilesystems
|
||||
|
||||
p.fsPruners = make([]*FSPruner, len(res.filesystems))
|
||||
_, add, end := trace.WithTaskGroup(ctx, "pruner-fan-out-fs")
|
||||
for i, fs := range res.filesystems {
|
||||
p.fsPruners[i] = NewFSPruner(p.jid, p.side, p.keepRules, fs)
|
||||
add(func(ctx context.Context) {
|
||||
p.fsPruners[i].Run(ctx)
|
||||
})
|
||||
}
|
||||
go func() {
|
||||
end()
|
||||
close(p.done)
|
||||
}()
|
||||
|
||||
case req := <-p.stopReqs:
|
||||
cancel()
|
||||
go func() {
|
||||
<-p.done
|
||||
close(req.complete)
|
||||
}()
|
||||
case req := <-p.reportReqs:
|
||||
req.reply <- p.report(req.ctx)
|
||||
case <-p.done:
|
||||
p.state = StateDone
|
||||
return p.report(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pruner) Report(ctx context.Context) *Report {
|
||||
req := reportRequest{
|
||||
ctx: ctx,
|
||||
reply: make(chan *Report, 1),
|
||||
}
|
||||
select {
|
||||
case p.reportReqs <- req:
|
||||
return <-req.reply
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-p.done:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pruner) report(ctx context.Context) *Report {
|
||||
fsreports := make([]*FSReport, len(p.fsPruners))
|
||||
for i := range fsreports {
|
||||
fsreports[i] = p.fsPruners[i].report()
|
||||
}
|
||||
return &Report{
|
||||
State: p.state,
|
||||
ListFilesystemsError: p.listFilesystemsError,
|
||||
Filesystems: fsreports,
|
||||
}
|
||||
}
|
||||
|
||||
// implements pruning.Snapshot
|
||||
type snapshot struct {
|
||||
replicated bool
|
||||
stepHolds []pruning.StepHold
|
||||
zfs.FilesystemVersion
|
||||
|
||||
state SnapState
|
||||
destroyOp *zfs.DestroySnapOp
|
||||
}
|
||||
|
||||
//go:generate enumer -type=SnapState -json
|
||||
type SnapState int
|
||||
|
||||
const (
|
||||
SnapStateInitialized SnapState = iota
|
||||
SnapStateKeeping
|
||||
SnapStateDeletePending
|
||||
SnapStateDeleteAttempted
|
||||
)
|
||||
|
||||
// implements pruning.StepHold
|
||||
type stepHold struct {
|
||||
endpoint.Abstraction
|
||||
}
|
||||
|
||||
func (s snapshot) Replicated() bool { return s.replicated }
|
||||
func (s snapshot) StepHolds() []pruning.StepHold { return s.stepHolds }
|
||||
|
||||
func (s stepHold) GetJobID() endpoint.JobID { return *s.Abstraction.GetJobID() }
|
||||
|
||||
type FSPruner struct {
|
||||
jid endpoint.JobID
|
||||
side Side
|
||||
keepRules []pruning.KeepRule
|
||||
fsp *zfs.DatasetPath
|
||||
|
||||
state FSState
|
||||
|
||||
// all channels consumed by the run loop
|
||||
planned chan fsPlanRes
|
||||
executed chan fsExecuteRes
|
||||
done chan struct{}
|
||||
reportReqs chan fsReportReq
|
||||
|
||||
keepList []*snapshot // valid in FSStateExecuting and forward
|
||||
destroyList []*snapshot // valid in FSStateExecuting and forward, field .destroyOp is invalid until FSStateExecuting is left
|
||||
|
||||
}
|
||||
|
||||
type fsPlanRes struct {
|
||||
keepList []*snapshot
|
||||
destroyList []*snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
type fsExecuteRes struct {
|
||||
completedDestroyOps []*zfs.DestroySnapOp // same len() as FSPruner.destroyList
|
||||
}
|
||||
|
||||
type fsReportReq struct {
|
||||
res chan *FSReport
|
||||
}
|
||||
|
||||
type FSReport struct {
|
||||
State FSState
|
||||
KeepList []*SnapReport
|
||||
Destroy []*SnapReport
|
||||
}
|
||||
|
||||
type SnapReport struct {
|
||||
State SnapState
|
||||
Name string
|
||||
Replicated bool
|
||||
StepHoldCount int
|
||||
DestroyError error
|
||||
}
|
||||
|
||||
//go:generate enumer -type=FSState -json
|
||||
type FSState int
|
||||
|
||||
const (
|
||||
FSStateInitialized FSState = iota
|
||||
FSStatePlanning
|
||||
FSStatePlanErr
|
||||
FSStateExecuting
|
||||
FSStateExecuteErr
|
||||
FSStateExecuteSuccess
|
||||
)
|
||||
|
||||
func (s FSState) IsTerminal() bool {
|
||||
return s == FSStatePlanErr || s == FSStateExecuteErr || s == FSStateExecuteSuccess
|
||||
}
|
||||
|
||||
func NewFSPruner(jid endpoint.JobID, side Side, keepRules []pruning.KeepRule, fsp *zfs.DatasetPath) *FSPruner {
|
||||
return &FSPruner{
|
||||
jid, side, keepRules, fsp,
|
||||
FSStateInitialized,
|
||||
make(chan fsPlanRes),
|
||||
make(chan fsExecuteRes),
|
||||
make(chan struct{}),
|
||||
make(chan fsReportReq),
|
||||
nil, nil,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *FSPruner) Run(ctx context.Context) *FSReport {
|
||||
|
||||
defer func() {
|
||||
}()
|
||||
|
||||
p.state = FSStatePlanning
|
||||
|
||||
go func() { p.planned <- p.plan(ctx) }()
|
||||
|
||||
out:
|
||||
for !p.state.IsTerminal() {
|
||||
select {
|
||||
case res := <-p.planned:
|
||||
|
||||
if res.err != nil {
|
||||
p.state = FSStatePlanErr
|
||||
continue
|
||||
}
|
||||
p.state = FSStateExecuting
|
||||
p.keepList = res.keepList
|
||||
p.destroyList = res.destroyList
|
||||
|
||||
go func() { p.executed <- p.execute(ctx, p.destroyList) }()
|
||||
|
||||
case res := <-p.executed:
|
||||
|
||||
if len(res.completedDestroyOps) != len(p.destroyList) {
|
||||
panic("impl error: completedDestroyOps is a vector corresponding to entries in p.destroyList")
|
||||
}
|
||||
|
||||
var erronous []*zfs.DestroySnapOp
|
||||
for i, op := range res.completedDestroyOps {
|
||||
if *op.ErrOut != nil {
|
||||
erronous = append(erronous, op)
|
||||
}
|
||||
p.destroyList[i].destroyOp = op
|
||||
p.destroyList[i].state = SnapStateDeleteAttempted
|
||||
}
|
||||
if len(erronous) > 0 {
|
||||
p.state = FSStateExecuteErr
|
||||
} else {
|
||||
p.state = FSStateExecuteSuccess
|
||||
}
|
||||
|
||||
close(p.done)
|
||||
|
||||
case <-p.reportReqs:
|
||||
panic("unimp")
|
||||
case <-p.done:
|
||||
break out
|
||||
}
|
||||
}
|
||||
|
||||
// TODO render last FS report
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *FSPruner) plan(ctx context.Context) fsPlanRes {
|
||||
fs := p.fsp.ToString()
|
||||
vs, err := zfs.ZFSListFilesystemVersions(ctx, p.fsp, zfs.ListFilesystemVersionsOptions{})
|
||||
if err != nil {
|
||||
return fsPlanRes{err: errors.Wrap(err, "list filesystem versions")}
|
||||
}
|
||||
|
||||
allJobsStepHolds, absErrs, err := endpoint.ListAbstractions(ctx, endpoint.ListZFSHoldsAndBookmarksQuery{
|
||||
FS: endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
||||
FS: &fs,
|
||||
},
|
||||
What: endpoint.AbstractionTypeSet{
|
||||
endpoint.AbstractionStepHold: true,
|
||||
},
|
||||
Concurrency: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return fsPlanRes{err: errors.Wrap(err, "list abstractions")}
|
||||
}
|
||||
if len(absErrs) > 0 {
|
||||
logging.GetLogger(ctx, logging.SubsysPruning).WithError(endpoint.ListAbstractionsErrors(absErrs)).
|
||||
Error("error listing some step holds, prune attempt might fail with 'dataset is busy' errors")
|
||||
}
|
||||
|
||||
repPos, err := p.side.GetReplicationPosition(ctx, p.fsp.ToString())
|
||||
if err != nil {
|
||||
return fsPlanRes{err: errors.Wrap(err, "get replication position")}
|
||||
}
|
||||
|
||||
vsAsSnaps := make([]pruning.Snapshot, len(vs))
|
||||
for i := range vs {
|
||||
var repPosCreateTxgOrZero uint64
|
||||
if repPos != nil {
|
||||
repPosCreateTxgOrZero = repPos.GetCreateTXG()
|
||||
}
|
||||
s := &snapshot{
|
||||
state: SnapStateInitialized,
|
||||
FilesystemVersion: vs[i],
|
||||
replicated: vs[i].GetCreateTXG() <= repPosCreateTxgOrZero,
|
||||
}
|
||||
for _, h := range allJobsStepHolds {
|
||||
if zfs.FilesystemVersionEqualIdentity(vs[i], h.GetFilesystemVersion()) {
|
||||
s.stepHolds = append(s.stepHolds, stepHold{h})
|
||||
}
|
||||
}
|
||||
vsAsSnaps[i] = s
|
||||
}
|
||||
|
||||
downcastToSnapshots := func(l []pruning.Snapshot) (r []*snapshot) {
|
||||
r = make([]*snapshot, len(l))
|
||||
for i, e := range l {
|
||||
r[i] = e.(*snapshot)
|
||||
}
|
||||
return r
|
||||
}
|
||||
pruningResult := pruning.PruneSnapshots(vsAsSnaps, p.keepRules)
|
||||
remove, keep := downcastToSnapshots(pruningResult.Remove), downcastToSnapshots(pruningResult.Keep)
|
||||
if len(remove)+len(keep) != len(vsAsSnaps) {
|
||||
for _, s := range vsAsSnaps {
|
||||
r, _ := json.MarshalIndent(s.(*snapshot).report(), "", " ")
|
||||
fmt.Fprintf(os.Stderr, "%s\n", string(r))
|
||||
}
|
||||
panic("indecisive")
|
||||
}
|
||||
|
||||
for _, s := range remove {
|
||||
s.state = SnapStateDeletePending
|
||||
}
|
||||
for _, s := range keep {
|
||||
s.state = SnapStateKeeping
|
||||
}
|
||||
|
||||
return fsPlanRes{keepList: keep, destroyList: remove, err: nil}
|
||||
}
|
||||
|
||||
func (p *FSPruner) execute(ctx context.Context, destroyList []*snapshot) fsExecuteRes {
|
||||
ops := make([]*zfs.DestroySnapOp, len(destroyList))
|
||||
for i, fsv := range p.destroyList {
|
||||
ops[i] = &zfs.DestroySnapOp{
|
||||
Filesystem: p.fsp.ToString(),
|
||||
Name: fsv.GetName(),
|
||||
ErrOut: new(error),
|
||||
}
|
||||
}
|
||||
zfs.ZFSDestroyFilesystemVersions(ctx, ops)
|
||||
|
||||
return fsExecuteRes{completedDestroyOps: ops}
|
||||
}
|
||||
|
||||
func (p *FSPruner) report() *FSReport {
|
||||
return &FSReport{
|
||||
State: p.state,
|
||||
KeepList: p.reportRenderSnapReports(p.keepList),
|
||||
Destroy: p.reportRenderSnapReports(p.destroyList),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *FSPruner) reportRenderSnapReports(l []*snapshot) (r []*SnapReport) {
|
||||
r = make([]*SnapReport, len(l))
|
||||
for i := range l {
|
||||
r[i] = l[i].report()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (s *snapshot) report() *SnapReport {
|
||||
var snapErr error
|
||||
if s.state == SnapStateDeleteAttempted {
|
||||
if *s.destroyOp.ErrOut != nil {
|
||||
snapErr = (*s.destroyOp.ErrOut)
|
||||
}
|
||||
}
|
||||
return &SnapReport{
|
||||
State: s.state,
|
||||
Name: s.Name,
|
||||
Replicated: s.Replicated(),
|
||||
StepHoldCount: len(s.stepHolds),
|
||||
DestroyError: snapErr,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user