adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP endpoint abstractions + pruning integration / pruner rewrite

Browse Source
This commit is contained in:
Christian Schwarz
2020-05-29 00:09:43 +02:00
parent f0146d03d3
commit 4a0104a44f
26 changed files with 1672 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pruning/keep_grid.go
+16 -9
View File
@@ -66,24 +66,26 @@ type retentionGridAdaptor struct {
Snapshot
}
func (a retentionGridAdaptor) Date() time.Time { return a.Snapshot.GetCreation() }
func (a retentionGridAdaptor) LessThan(b retentiongrid.Entry) bool {
return a.Date().Before(b.Date())
}
// Prune filters snapshots with the retention grid.
func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
func (p *KeepGrid) KeepRule(snaps []Snapshot) PruneSnapshotsResult {
snaps = filterSnapList(snaps, func(snapshot Snapshot) bool {
return p.re.MatchString(snapshot.Name())
reCandidates := partitionSnapList(snaps, func(snapshot Snapshot) bool {
return p.re.MatchString(snapshot.GetName())
})
if len(snaps) == 0 {
return nil
if len(reCandidates.Remove) == 0 {
return reCandidates
}
// Build adaptors for retention grid
adaptors := make([]retentiongrid.Entry, 0)
for i := range snaps {
adaptors = append(adaptors, retentionGridAdaptor{snaps[i]})
adaptors = append(adaptors, retentionGridAdaptor{reCandidates.Remove[i]})
}
// determine 'now' edge
@@ -93,12 +95,17 @@ func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
now := adaptors[len(adaptors)-1].Date()
// Evaluate retention grid
_, removea := p.retentionGrid.FitEntries(now, adaptors)
keepa, removea := p.retentionGrid.FitEntries(now, adaptors)
// Revert adaptors
destroyList = make([]Snapshot, len(removea))
destroyList := make([]Snapshot, len(removea))
for i := range removea {
destroyList[i] = removea[i].(retentionGridAdaptor).Snapshot
}
return destroyList
for _, a := range keepa {
reCandidates.Keep = append(reCandidates.Keep, a.(retentionGridAdaptor))
}
reCandidates.Remove = destroyList
return reCandidates
}
pruning/keep_helpers.go
+7 -4
View File
@@ -1,10 +1,13 @@
package pruning
func filterSnapList(snaps []Snapshot, predicate func(Snapshot) bool) []Snapshot {
r := make([]Snapshot, 0, len(snaps))
func partitionSnapList(snaps []Snapshot, remove func(Snapshot) bool) (r PruneSnapshotsResult) {
r.Keep = make([]Snapshot, 0, len(snaps))
r.Remove = make([]Snapshot, 0, len(snaps))
for i := range snaps {
if predicate(snaps[i]) {
r = append(r, snaps[i])
if remove(snaps[i]) {
r.Remove = append(r.Remove, snaps[i])
} else {
r.Keep = append(r.Keep, snaps[i])
}
}
return r
pruning/keep_last_n.go
+4 -4
View File
@@ -17,17 +17,17 @@ func NewKeepLastN(n int) (*KeepLastN, error) {
return &KeepLastN{n}, nil
}
func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
func (k KeepLastN) KeepRule(snaps []Snapshot) PruneSnapshotsResult {
if k.n > len(snaps) {
return []Snapshot{}
return PruneSnapshotsResult{Keep: snaps}
}
res := shallowCopySnapList(snaps)
sort.Slice(res, func(i, j int) bool {
return res[i].Date().After(res[j].Date())
return res[i].GetCreateTXG() > res[j].GetCreateTXG()
})
return res[k.n:]
return PruneSnapshotsResult{Remove: res[k.n:], Keep: res[:k.n]}
}
pruning/keep_not_replicated.go
+2 -2
View File
@@ -2,8 +2,8 @@ package pruning
type KeepNotReplicated struct{}
func (*KeepNotReplicated) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
return filterSnapList(snaps, func(snapshot Snapshot) bool {
func (*KeepNotReplicated) KeepRule(snaps []Snapshot) PruneSnapshotsResult {
return partitionSnapList(snaps, func(snapshot Snapshot) bool {
return snapshot.Replicated()
})
}
pruning/keep_regex.go
+4 -4
View File
@@ -27,12 +27,12 @@ func MustKeepRegex(expr string, negate bool) *KeepRegex {
return k
}
func (k *KeepRegex) KeepRule(snaps []Snapshot) []Snapshot {
return filterSnapList(snaps, func(s Snapshot) bool {
func (k *KeepRegex) KeepRule(snaps []Snapshot) PruneSnapshotsResult {
return partitionSnapList(snaps, func(s Snapshot) bool {
if k.negate {
return k.expr.FindStringIndex(s.Name()) != nil
return k.expr.FindStringIndex(s.GetName()) != nil
} else {
return k.expr.FindStringIndex(s.Name()) == nil
return k.expr.FindStringIndex(s.GetName()) == nil
}
})
}
pruning/keep_step_holds.go
+44
View File
@@ -0,0 +1,44 @@
package pruning
import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/endpoint"
)
type KeepStepHolds struct {
keepJobIDs map[endpoint.JobID]bool
}
var _ KeepRule = (*KeepStepHolds)(nil)
func NewKeepStepHolds(mainJobId endpoint.JobID, additionalJobIdsStrings []string) (_ *KeepStepHolds, err error) {
additionalJobIds := make(map[endpoint.JobID]bool, len(additionalJobIdsStrings))
mainJobId.MustValidate()
additionalJobIds[mainJobId] = true
for i := range additionalJobIdsStrings {
ajid, err := endpoint.MakeJobID(additionalJobIdsStrings[i])
if err != nil {
return nil, errors.WithMessagef(err, "cannot parse job id %q: %s", additionalJobIdsStrings[i])
}
if additionalJobIds[ajid] == true {
return nil, errors.Errorf("duplicate job id %q", ajid)
}
}
return &KeepStepHolds{additionalJobIds}, nil
}
func (h *KeepStepHolds) KeepRule(snaps []Snapshot) PruneSnapshotsResult {
return partitionSnapList(snaps, func(s Snapshot) bool {
holdingJobIDs := make(map[endpoint.JobID]bool)
for _, h := range s.StepHolds() {
holdingJobIDs[h.GetJobID()] = true
}
oneOrMoreOfOurJobIDsHoldsSnap := false
for kjid := range h.keepJobIDs {
oneOrMoreOfOurJobIDsHoldsSnap = oneOrMoreOfOurJobIDsHoldsSnap || holdingJobIDs[kjid]
}
return !oneOrMoreOfOurJobIDsHoldsSnap
})
}
pruning/pruning.go
+65 -17
View File
@@ -7,47 +7,93 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/endpoint"
)
type KeepRule interface {
KeepRule(snaps []Snapshot) (destroyList []Snapshot)
KeepRule(snaps []Snapshot) PruneSnapshotsResult
}
type Snapshot interface {
Name() string
GetName() string
Replicated() bool
Date() time.Time
GetCreation() time.Time
GetCreateTXG() uint64
StepHolds() []StepHold
}
// The returned snapshot list is guaranteed to only contains elements of input parameter snaps
func PruneSnapshots(snaps []Snapshot, keepRules []KeepRule) []Snapshot {
type StepHold interface {
GetJobID() endpoint.JobID
}
type PruneSnapshotsResult struct {
Remove, Keep []Snapshot
}
// The returned snapshot results are a partition of the snaps argument.
// That means than len(Remove) + len(Keep) == len(snaps)
func PruneSnapshots(snapsI []Snapshot, keepRules []KeepRule) PruneSnapshotsResult {
if len(keepRules) == 0 {
return []Snapshot{}
return PruneSnapshotsResult{Remove: nil, Keep: snapsI}
}
type snapshot struct {
Snapshot
keepCount, removeCount int
}
// project down to snapshot
snaps := make([]Snapshot, len(snapsI))
for i := range snaps {
snaps[i] = &snapshot{snapsI[i], 0, 0}
}
remCount := make(map[Snapshot]int, len(snaps))
for _, r := range keepRules {
ruleRems := r.KeepRule(snaps)
for _, ruleRem := range ruleRems {
remCount[ruleRem]++
ruleImplCheckSet := make(map[Snapshot]int, len(snaps))
for _, s := range snaps {
ruleImplCheckSet[s] = ruleImplCheckSet[s] + 1
}
ruleResults := r.KeepRule(snaps)
for _, s := range snaps {
ruleImplCheckSet[s] = ruleImplCheckSet[s] - 1
}
for _, n := range ruleImplCheckSet {
if n != 0 {
panic(fmt.Sprintf("incorrect rule implementation: %T", r))
}
}
for _, s := range ruleResults.Remove {
s.(*snapshot).removeCount++
}
for _, s := range ruleResults.Keep {
s.(*snapshot).keepCount++
}
}
remove := make([]Snapshot, 0, len(snaps))
for snap, rc := range remCount {
if rc == len(keepRules) {
remove = append(remove, snap)
keep := make([]Snapshot, 0, len(snaps))
for _, sI := range snaps {
s := sI.(*snapshot)
if s.removeCount == len(keepRules) {
// all keep rules agree to remove the snap
remove = append(remove, s.Snapshot)
} else {
keep = append(keep, s.Snapshot)
}
}
return remove
return PruneSnapshotsResult{Remove: remove, Keep: keep}
}
func RulesFromConfig(in []config.PruningEnum) (rules []KeepRule, err error) {
func RulesFromConfig(mainJobId endpoint.JobID, in []config.PruningEnum) (rules []KeepRule, err error) {
rules = make([]KeepRule, len(in))
for i := range in {
rules[i], err = RuleFromConfig(in[i])
rules[i], err = RuleFromConfig(mainJobId, in[i])
if err != nil {
return nil, errors.Wrapf(err, "cannot build rule #%d", i)
}
@@ -55,7 +101,7 @@ func RulesFromConfig(in []config.PruningEnum) (rules []KeepRule, err error) {
return rules, nil
}
func RuleFromConfig(in config.PruningEnum) (KeepRule, error) {
func RuleFromConfig(mainJobId endpoint.JobID, in config.PruningEnum) (KeepRule, error) {
switch v := in.Ret.(type) {
case *config.PruneKeepNotReplicated:
return NewKeepNotReplicated(), nil
@@ -65,6 +111,8 @@ func RuleFromConfig(in config.PruningEnum) (KeepRule, error) {
return NewKeepRegex(v.Regex, v.Negate)
case *config.PruneGrid:
return NewKeepGrid(v)
case *config.PruneKeepStepHolds:
return NewKeepStepHolds(mainJobId, v.AdditionalJobIds)
default:
return nil, fmt.Errorf("unknown keep rule type %T", v)
}