move implementation to internal/ directory (#828)
This commit is contained in:
Christian Schwarz
committed by
GitHub
GitHub
parent
b9b9ad10cf
commit
908807bd59
@@ -0,0 +1,116 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/pruning/retentiongrid"
|
||||
)
|
||||
|
||||
// KeepGrid fits snapshots that match a given regex into a retentiongrid.Grid,
|
||||
// uses the most recent snapshot among those that match the regex as 'now',
|
||||
// and deletes all snapshots that do not fit the grid specification.
|
||||
type KeepGrid struct {
|
||||
retentionGrid *retentiongrid.Grid
|
||||
re *regexp.Regexp
|
||||
}
|
||||
|
||||
func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) {
|
||||
|
||||
if in.Regex == "" {
|
||||
return nil, fmt.Errorf("Regex must not be empty")
|
||||
}
|
||||
re, err := regexp.Compile(in.Regex)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Regex is invalid")
|
||||
}
|
||||
|
||||
return newKeepGrid(re, in.Grid)
|
||||
}
|
||||
|
||||
func MustNewKeepGrid(regex, gridspec string) *KeepGrid {
|
||||
|
||||
ris, err := config.ParseRetentionIntervalSpec(gridspec)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(regex)
|
||||
|
||||
grid, err := newKeepGrid(re, ris)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return grid
|
||||
}
|
||||
|
||||
func newKeepGrid(re *regexp.Regexp, configIntervals []config.RetentionInterval) (*KeepGrid, error) {
|
||||
if re == nil {
|
||||
panic("re must not be nil")
|
||||
}
|
||||
|
||||
if len(configIntervals) == 0 {
|
||||
return nil, errors.New("retention grid must specify at least one interval")
|
||||
}
|
||||
|
||||
intervals := make([]retentiongrid.Interval, len(configIntervals))
|
||||
for i := range configIntervals {
|
||||
intervals[i] = &configIntervals[i]
|
||||
}
|
||||
|
||||
// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
|
||||
lastDuration := time.Duration(0)
|
||||
for i := range intervals {
|
||||
|
||||
if intervals[i].Length() < lastDuration {
|
||||
// If all intervals before were keep=all, this is ok
|
||||
allPrevKeepCountAll := true
|
||||
for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
|
||||
allPrevKeepCountAll = intervals[j].KeepCount() == config.RetentionGridKeepCountAll
|
||||
}
|
||||
if allPrevKeepCountAll {
|
||||
goto isMonotonicIncrease
|
||||
}
|
||||
return nil, errors.New("retention grid interval length must be monotonically increasing")
|
||||
}
|
||||
isMonotonicIncrease:
|
||||
lastDuration = intervals[i].Length()
|
||||
}
|
||||
|
||||
return &KeepGrid{
|
||||
retentionGrid: retentiongrid.NewGrid(intervals),
|
||||
re: re,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Prune filters snapshots with the retention grid.
|
||||
func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
|
||||
matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
return p.re.MatchString(snapshot.Name())
|
||||
})
|
||||
|
||||
// snaps that don't match the regex are not kept by this rule
|
||||
destroyList = append(destroyList, notMatching...)
|
||||
|
||||
if len(matching) == 0 {
|
||||
return destroyList
|
||||
}
|
||||
|
||||
// Evaluate retention grid
|
||||
entrySlice := make([]retentiongrid.Entry, 0)
|
||||
for i := range matching {
|
||||
entrySlice = append(entrySlice, matching[i])
|
||||
}
|
||||
_, gridDestroyList := p.retentionGrid.FitEntries(entrySlice)
|
||||
|
||||
// Revert adaptors
|
||||
for i := range gridDestroyList {
|
||||
destroyList = append(destroyList, gridDestroyList[i].(Snapshot))
|
||||
}
|
||||
return destroyList
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package pruning
|
||||
|
||||
func filterSnapList(snaps []Snapshot, predicate func(Snapshot) bool) []Snapshot {
|
||||
r := make([]Snapshot, 0, len(snaps))
|
||||
for i := range snaps {
|
||||
if predicate(snaps[i]) {
|
||||
r = append(r, snaps[i])
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func partitionSnapList(snaps []Snapshot, predicate func(Snapshot) bool) (sTrue, sFalse []Snapshot) {
|
||||
for i := range snaps {
|
||||
if predicate(snaps[i]) {
|
||||
sTrue = append(sTrue, snaps[i])
|
||||
} else {
|
||||
sFalse = append(sFalse, snaps[i])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func shallowCopySnapList(snaps []Snapshot) []Snapshot {
|
||||
c := make([]Snapshot, len(snaps))
|
||||
copy(c, snaps)
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestShallowCopySnapList(t *testing.T) {
|
||||
|
||||
l1 := []Snapshot{
|
||||
stubSnap{name: "foo"},
|
||||
stubSnap{name: "bar"},
|
||||
}
|
||||
l2 := shallowCopySnapList(l1)
|
||||
|
||||
assert.Equal(t, l1, l2)
|
||||
|
||||
l1[0] = stubSnap{name: "baz"}
|
||||
assert.Equal(t, "baz", l1[0].Name())
|
||||
assert.Equal(t, "foo", l2[0].Name())
|
||||
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type KeepLastN struct {
|
||||
n int
|
||||
re *regexp.Regexp
|
||||
}
|
||||
|
||||
func MustKeepLastN(n int, regex string) *KeepLastN {
|
||||
k, err := NewKeepLastN(n, regex)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func NewKeepLastN(n int, regex string) (*KeepLastN, error) {
|
||||
if n <= 0 {
|
||||
return nil, errors.Errorf("must specify positive number as 'keep last count', got %d", n)
|
||||
}
|
||||
re, err := regexp.Compile(regex)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("invalid regex %q: %s", regex, err)
|
||||
}
|
||||
return &KeepLastN{n, re}, nil
|
||||
}
|
||||
|
||||
func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
return k.re.MatchString(snapshot.Name())
|
||||
})
|
||||
// snaps that don't match the regex are not kept by this rule
|
||||
destroyList = append(destroyList, notMatching...)
|
||||
|
||||
if len(matching) == 0 {
|
||||
return destroyList
|
||||
}
|
||||
|
||||
sort.Slice(matching, func(i, j int) bool {
|
||||
// by date (youngest first)
|
||||
id, jd := matching[i].Date(), matching[j].Date()
|
||||
if !id.Equal(jd) {
|
||||
return id.After(jd)
|
||||
}
|
||||
// then lexicographically descending (e.g. b, a)
|
||||
return strings.Compare(matching[i].Name(), matching[j].Name()) == 1
|
||||
})
|
||||
|
||||
n := k.n
|
||||
if n > len(matching) {
|
||||
n = len(matching)
|
||||
}
|
||||
destroyList = append(destroyList, matching[n:]...)
|
||||
return destroyList
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestKeepLastN(t *testing.T) {
|
||||
|
||||
o := func(minutes int) time.Time {
|
||||
return time.Unix(123, 0).Add(time.Duration(minutes) * time.Minute)
|
||||
}
|
||||
|
||||
inputs := map[string][]Snapshot{
|
||||
"s1": []Snapshot{
|
||||
stubSnap{name: "1", date: o(10)},
|
||||
stubSnap{name: "2", date: o(20)},
|
||||
stubSnap{name: "3", date: o(15)},
|
||||
stubSnap{name: "4", date: o(30)},
|
||||
stubSnap{name: "5", date: o(30)},
|
||||
},
|
||||
"s2": []Snapshot{},
|
||||
}
|
||||
|
||||
tcs := map[string]testCase{
|
||||
"keep2": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(2, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"1": true, "2": true, "3": true,
|
||||
},
|
||||
},
|
||||
"keep1OfTwoWithSameTime": { // Keep one of two with same time
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(1, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{"1": true, "2": true, "3": true, "4": true},
|
||||
},
|
||||
"keepMany": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(100, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"empty_input": {
|
||||
inputs: inputs["s2"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(100, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"empty_regex": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(4, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"1": true,
|
||||
},
|
||||
},
|
||||
"multiple_regexes": {
|
||||
inputs: []Snapshot{
|
||||
stubSnap{"a1", false, o(10)},
|
||||
stubSnap{"b1", false, o(11)},
|
||||
stubSnap{"a2", false, o(20)},
|
||||
stubSnap{"b2", false, o(21)},
|
||||
stubSnap{"a3", false, o(30)},
|
||||
stubSnap{"b3", false, o(31)},
|
||||
},
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(2, "^a"),
|
||||
MustKeepLastN(2, "^b"),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"a1": true,
|
||||
"b1": true,
|
||||
},
|
||||
},
|
||||
"keep_more_than_matching": {
|
||||
inputs: []Snapshot{
|
||||
stubSnap{"a1", false, o(10)},
|
||||
stubSnap{"b1", false, o(11)},
|
||||
stubSnap{"a2", false, o(12)},
|
||||
},
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(4, "a"),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"b1": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testTable(tcs, t)
|
||||
|
||||
t.Run("mustBePositive", func(t *testing.T) {
|
||||
var err error
|
||||
_, err = NewKeepLastN(0, "foo")
|
||||
assert.Error(t, err)
|
||||
_, err = NewKeepLastN(-5, "foo")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("emptyRegexAllowed", func(t *testing.T) {
|
||||
_, err := NewKeepLastN(23, "")
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package pruning
|
||||
|
||||
type KeepNotReplicated struct{}
|
||||
|
||||
func (*KeepNotReplicated) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
return filterSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
return snapshot.Replicated()
|
||||
})
|
||||
}
|
||||
|
||||
func NewKeepNotReplicated() *KeepNotReplicated {
|
||||
return &KeepNotReplicated{}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewKeepNotReplicated(t *testing.T) {
|
||||
|
||||
inputs := map[string][]Snapshot{
|
||||
"s1": []Snapshot{
|
||||
stubSnap{name: "1", replicated: true},
|
||||
stubSnap{name: "2", replicated: false},
|
||||
stubSnap{name: "3", replicated: true},
|
||||
},
|
||||
"s2": []Snapshot{},
|
||||
}
|
||||
|
||||
tcs := map[string]testCase{
|
||||
"destroysOnlyReplicated": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
NewKeepNotReplicated(),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"1": true, "3": true,
|
||||
},
|
||||
},
|
||||
"empty": {
|
||||
inputs: inputs["s2"],
|
||||
rules: []KeepRule{
|
||||
NewKeepNotReplicated(),
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
}
|
||||
|
||||
testTable(tcs, t)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
)
|
||||
|
||||
type KeepRegex struct {
|
||||
expr *regexp.Regexp
|
||||
negate bool
|
||||
}
|
||||
|
||||
var _ KeepRule = &KeepRegex{}
|
||||
|
||||
func NewKeepRegex(expr string, negate bool) (*KeepRegex, error) {
|
||||
re, err := regexp.Compile(expr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &KeepRegex{re, negate}, nil
|
||||
}
|
||||
|
||||
func MustKeepRegex(expr string, negate bool) *KeepRegex {
|
||||
k, err := NewKeepRegex(expr, negate)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func (k *KeepRegex) KeepRule(snaps []Snapshot) []Snapshot {
|
||||
return filterSnapList(snaps, func(s Snapshot) bool {
|
||||
if k.negate {
|
||||
return k.expr.FindStringIndex(s.Name()) != nil
|
||||
} else {
|
||||
return k.expr.FindStringIndex(s.Name()) == nil
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestKeepRegexNegation(t *testing.T) {
|
||||
|
||||
noneg := MustKeepRegex("^zrepl_", false)
|
||||
neg := MustKeepRegex("^zrepl_", true)
|
||||
|
||||
snaps := []Snapshot{
|
||||
stubSnap{name: "zrepl_foobar"},
|
||||
stubSnap{name: "zrepl"},
|
||||
stubSnap{name: "barfoo"},
|
||||
}
|
||||
|
||||
destroyNonNeg := snapshotList(noneg.KeepRule(snaps))
|
||||
t.Logf("non-negated rule destroys: %#v", destroyNonNeg.NameList())
|
||||
assert.True(t, destroyNonNeg.ContainsName("zrepl"))
|
||||
assert.True(t, destroyNonNeg.ContainsName("barfoo"))
|
||||
assert.False(t, destroyNonNeg.ContainsName("zrepl_foobar"))
|
||||
|
||||
destroyNeg := snapshotList(neg.KeepRule(snaps))
|
||||
t.Logf("negated rule destroys: %#v", destroyNeg.NameList())
|
||||
assert.False(t, destroyNeg.ContainsName("zrepl"))
|
||||
assert.False(t, destroyNeg.ContainsName("barfoo"))
|
||||
assert.True(t, destroyNeg.ContainsName("zrepl_foobar"))
|
||||
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
)
|
||||
|
||||
type KeepRule interface {
|
||||
KeepRule(snaps []Snapshot) (destroyList []Snapshot)
|
||||
}
|
||||
|
||||
type Snapshot interface {
|
||||
Name() string
|
||||
Replicated() bool
|
||||
Date() time.Time
|
||||
}
|
||||
|
||||
// The returned snapshot list is guaranteed to only contains elements of input parameter snaps
|
||||
func PruneSnapshots(snaps []Snapshot, keepRules []KeepRule) []Snapshot {
|
||||
|
||||
if len(keepRules) == 0 {
|
||||
return []Snapshot{}
|
||||
}
|
||||
|
||||
remCount := make(map[Snapshot]int, len(snaps))
|
||||
for _, r := range keepRules {
|
||||
ruleRems := r.KeepRule(snaps)
|
||||
for _, ruleRem := range ruleRems {
|
||||
remCount[ruleRem]++
|
||||
}
|
||||
}
|
||||
|
||||
remove := make([]Snapshot, 0, len(snaps))
|
||||
for snap, rc := range remCount {
|
||||
if rc == len(keepRules) {
|
||||
remove = append(remove, snap)
|
||||
}
|
||||
}
|
||||
|
||||
return remove
|
||||
}
|
||||
|
||||
func RulesFromConfig(in []config.PruningEnum) (rules []KeepRule, err error) {
|
||||
rules = make([]KeepRule, len(in))
|
||||
for i := range in {
|
||||
rules[i], err = RuleFromConfig(in[i])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot build rule #%d", i)
|
||||
}
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
|
||||
func RuleFromConfig(in config.PruningEnum) (KeepRule, error) {
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.PruneKeepNotReplicated:
|
||||
return NewKeepNotReplicated(), nil
|
||||
case *config.PruneKeepLastN:
|
||||
return NewKeepLastN(v.Count, v.Regex)
|
||||
case *config.PruneKeepRegex:
|
||||
return NewKeepRegex(v.Regex, v.Negate)
|
||||
case *config.PruneGrid:
|
||||
return NewKeepGrid(v)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown keep rule type %T", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type stubSnap struct {
|
||||
name string
|
||||
replicated bool
|
||||
date time.Time
|
||||
}
|
||||
|
||||
func (s stubSnap) Name() string { return s.name }
|
||||
|
||||
func (s stubSnap) Replicated() bool { return s.replicated }
|
||||
|
||||
func (s stubSnap) Date() time.Time { return s.date }
|
||||
|
||||
type testCase struct {
|
||||
inputs []Snapshot
|
||||
rules []KeepRule
|
||||
expDestroy map[string]bool
|
||||
}
|
||||
|
||||
type snapshotList []Snapshot
|
||||
|
||||
func (l snapshotList) ContainsName(n string) bool {
|
||||
for _, s := range l {
|
||||
if s.Name() == n {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l snapshotList) NameList() []string {
|
||||
res := make([]string, len(l))
|
||||
for i, s := range l {
|
||||
res[i] = s.Name()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func testTable(tcs map[string]testCase, t *testing.T) {
|
||||
for name := range tcs {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tc := tcs[name]
|
||||
destroyList := PruneSnapshots(tc.inputs, tc.rules)
|
||||
destroySet := make(map[string]bool, len(destroyList))
|
||||
for _, s := range destroyList {
|
||||
destroySet[s.Name()] = true
|
||||
}
|
||||
t.Logf("destroySet:\n%#v", destroySet)
|
||||
t.Logf("expected:\n%#v", tc.expDestroy)
|
||||
|
||||
require.Equal(t, len(tc.expDestroy), len(destroySet))
|
||||
for name := range destroySet {
|
||||
assert.True(t, tc.expDestroy[name], "%q", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneSnapshots(t *testing.T) {
|
||||
|
||||
inputs := map[string][]Snapshot{
|
||||
"s1": []Snapshot{
|
||||
stubSnap{name: "foo_123"},
|
||||
stubSnap{name: "foo_456"},
|
||||
stubSnap{name: "bar_123"},
|
||||
},
|
||||
}
|
||||
|
||||
reltime := func(secs int64) time.Time {
|
||||
return time.Unix(secs, 0)
|
||||
}
|
||||
|
||||
tcs := map[string]testCase{
|
||||
"simple": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepRegex("foo_", false),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"bar_123": true,
|
||||
},
|
||||
},
|
||||
"multipleRules": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepRegex("foo_", false),
|
||||
MustKeepRegex("bar_", false),
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"onlyThoseRemovedByAllAreRemoved": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepRegex("notInS1", false), // would remove all
|
||||
MustKeepRegex("bar_", false), // would remove all but bar_, i.e. foo_.*
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"foo_123": true,
|
||||
"foo_456": true,
|
||||
},
|
||||
},
|
||||
"noRulesKeepsAll": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"nilRulesKeepsAll": {
|
||||
inputs: inputs["s1"],
|
||||
rules: nil,
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"noSnaps": {
|
||||
inputs: []Snapshot{},
|
||||
rules: []KeepRule{
|
||||
MustKeepRegex("foo_", false),
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"multiple_grids_with_disjoint_regexes": {
|
||||
inputs: []Snapshot{
|
||||
stubSnap{"p1_a", false, reltime(4)},
|
||||
stubSnap{"p2_a", false, reltime(5)},
|
||||
stubSnap{"p1_b", false, reltime(14)},
|
||||
stubSnap{"p2_b", false, reltime(15)},
|
||||
stubSnap{"p1_c", false, reltime(29)},
|
||||
stubSnap{"p2_c", false, reltime(30)},
|
||||
},
|
||||
rules: []KeepRule{
|
||||
MustNewKeepGrid("^p1_", `1x10s | 1x10s`),
|
||||
MustNewKeepGrid("^p2_", `1x10s | 1x10s`),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"p1_a": true,
|
||||
"p2_a": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testTable(tcs, t)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package retentiongrid
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Interval interface {
|
||||
Length() time.Duration
|
||||
KeepCount() int
|
||||
}
|
||||
|
||||
const RetentionGridKeepCountAll int = -1
|
||||
|
||||
type Grid struct {
|
||||
intervals []Interval
|
||||
}
|
||||
|
||||
type Entry interface {
|
||||
Date() time.Time
|
||||
}
|
||||
|
||||
func NewGrid(l []Interval) *Grid {
|
||||
if len(l) == 0 {
|
||||
panic("must specify at least one interval")
|
||||
}
|
||||
// TODO Maybe check for ascending interval lengths here, although the algorithm
|
||||
// itself doesn't care about that.
|
||||
return &Grid{l}
|
||||
}
|
||||
|
||||
func (g Grid) FitEntries(entries []Entry) (keep, remove []Entry) {
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// determine 'now' based on youngest snapshot
|
||||
// => sort youngest-to-oldest
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
return entries[i].Date().After(entries[j].Date())
|
||||
})
|
||||
now := entries[0].Date()
|
||||
|
||||
return g.fitEntriesWithNow(now, entries)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
keepCount int
|
||||
youngerThan time.Time
|
||||
olderThanOrEq time.Time
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
func makeBucketFromInterval(olderThanOrEq time.Time, i Interval) bucket {
|
||||
var b bucket
|
||||
kc := i.KeepCount()
|
||||
if kc == 0 {
|
||||
panic("keep count 0 is not allowed")
|
||||
}
|
||||
if (kc < 0) && kc != RetentionGridKeepCountAll {
|
||||
panic("negative keep counts are not allowed")
|
||||
}
|
||||
b.keepCount = kc
|
||||
b.olderThanOrEq = olderThanOrEq
|
||||
b.youngerThan = b.olderThanOrEq.Add(-i.Length())
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *bucket) Contains(e Entry) bool {
|
||||
d := e.Date()
|
||||
olderThan := d.Before(b.olderThanOrEq)
|
||||
eq := d.Equal(b.olderThanOrEq)
|
||||
youngerThan := d.After(b.youngerThan)
|
||||
return (olderThan || eq) && youngerThan
|
||||
}
|
||||
|
||||
func (b *bucket) AddIfContains(e Entry) (added bool) {
|
||||
added = b.Contains(e)
|
||||
if added {
|
||||
b.entries = append(b.entries, e)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (b *bucket) RemoveYoungerSnapsExceedingKeepCount() (removed []Entry) {
|
||||
|
||||
if b.keepCount == RetentionGridKeepCountAll {
|
||||
return nil
|
||||
}
|
||||
|
||||
removeCount := len(b.entries) - b.keepCount
|
||||
if removeCount <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sort youngest-to-oldest
|
||||
sort.SliceStable(b.entries, func(i, j int) bool {
|
||||
return b.entries[i].Date().After(b.entries[j].Date())
|
||||
})
|
||||
|
||||
return b.entries[:removeCount]
|
||||
}
|
||||
|
||||
func (g Grid) fitEntriesWithNow(now time.Time, entries []Entry) (keep, remove []Entry) {
|
||||
|
||||
buckets := make([]bucket, len(g.intervals))
|
||||
|
||||
buckets[0] = makeBucketFromInterval(now, g.intervals[0])
|
||||
for i := 1; i < len(g.intervals); i++ {
|
||||
buckets[i] = makeBucketFromInterval(buckets[i-1].youngerThan, g.intervals[i])
|
||||
}
|
||||
|
||||
keep = make([]Entry, 0)
|
||||
remove = make([]Entry, 0)
|
||||
|
||||
assignEntriesToBuckets:
|
||||
for ei := 0; ei < len(entries); ei++ {
|
||||
e := entries[ei]
|
||||
// unconditionally keep entries that are in the future
|
||||
if now.Before(e.Date()) {
|
||||
keep = append(keep, e)
|
||||
continue assignEntriesToBuckets
|
||||
}
|
||||
// add to matching bucket, if any
|
||||
for bi := range buckets {
|
||||
if buckets[bi].AddIfContains(e) {
|
||||
continue assignEntriesToBuckets
|
||||
}
|
||||
}
|
||||
// unconditionally remove entries older than the oldest bucket
|
||||
remove = append(remove, e)
|
||||
}
|
||||
|
||||
// now apply the `KeepCount` per bucket
|
||||
for _, b := range buckets {
|
||||
destroy := b.RemoveYoungerSnapsExceedingKeepCount()
|
||||
remove = append(remove, destroy...)
|
||||
keep = append(keep, b.entries...)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package retentiongrid
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type testInterval struct {
|
||||
length time.Duration
|
||||
keepCount int
|
||||
}
|
||||
|
||||
func (i *testInterval) Length() time.Duration { return i.length }
|
||||
func (i *testInterval) KeepCount() int { return i.keepCount }
|
||||
|
||||
func gridFromString(gs string) (g *Grid) {
|
||||
sintervals := strings.Split(gs, "|")
|
||||
intervals := make([]Interval, len(sintervals))
|
||||
for idx, i := range sintervals {
|
||||
comps := strings.SplitN(i, ",", 2)
|
||||
var durationStr, numSnapsStr string
|
||||
durationStr = comps[0]
|
||||
if len(comps) == 1 {
|
||||
numSnapsStr = "1"
|
||||
} else {
|
||||
numSnapsStr = comps[1]
|
||||
}
|
||||
|
||||
var err error
|
||||
var interval testInterval
|
||||
|
||||
if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if interval.length, err = time.ParseDuration(durationStr); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
intervals[idx] = &interval
|
||||
}
|
||||
return NewGrid(intervals)
|
||||
}
|
||||
|
||||
type testSnap struct {
|
||||
Name string
|
||||
ShouldKeep bool
|
||||
date time.Time
|
||||
}
|
||||
|
||||
func (ds testSnap) Date() time.Time { return ds.date }
|
||||
|
||||
func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) {
|
||||
|
||||
snapDescr := func(d testSnap) string {
|
||||
return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now))
|
||||
}
|
||||
|
||||
t.Logf("keep list:\n")
|
||||
for k := range keep {
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
|
||||
}
|
||||
t.Logf("remove list:\n")
|
||||
for k := range remove {
|
||||
t.Logf("\t%s\n", snapDescr(remove[k].(testSnap)))
|
||||
}
|
||||
|
||||
t.Logf("\n\n")
|
||||
|
||||
for _, s := range input {
|
||||
d := s.(testSnap)
|
||||
descr := snapDescr(d)
|
||||
t.Logf("testing %s\n", descr)
|
||||
if d.ShouldKeep {
|
||||
assert.Contains(t, keep, d, "expecting %s to be kept", descr)
|
||||
} else {
|
||||
assert.Contains(t, remove, d, "expecting %s to be removed", descr)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("resulting list:\n")
|
||||
for k := range keep {
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInput(t *testing.T) {
|
||||
g := gridFromString("10m|10m|10m|1h")
|
||||
keep, remove := g.FitEntries([]Entry{})
|
||||
assert.Empty(t, keep)
|
||||
assert.Empty(t, remove)
|
||||
}
|
||||
|
||||
func TestIntervalBoundariesAndAlignment(t *testing.T) {
|
||||
g := gridFromString("10m|10m|10m")
|
||||
|
||||
t.Logf("%#v\n", g)
|
||||
|
||||
now := time.Unix(0, 0)
|
||||
|
||||
snaps := []Entry{
|
||||
testSnap{"0", true, now.Add(1 * time.Minute)}, // before now => keep unconditionally
|
||||
testSnap{"1", true, now}, // 1st interval left edge => inclusive
|
||||
testSnap{"2", true, now.Add(-10 * time.Minute)}, // 2nd interval left edge => inclusive
|
||||
testSnap{"3", true, now.Add(-20 * time.Minute)}, // 3rd interval left edge => inclusuive
|
||||
testSnap{"4", false, now.Add(-30 * time.Minute)}, // 3rd interval right edge => excludive
|
||||
testSnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval => remove unconditionally
|
||||
}
|
||||
|
||||
keep, remove := g.fitEntriesWithNow(now, snaps)
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
}
|
||||
|
||||
func TestKeepsOldestSnapsInABucket(t *testing.T) {
|
||||
g := gridFromString("1m,2")
|
||||
|
||||
relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
|
||||
|
||||
snaps := []Entry{
|
||||
testSnap{"1", true, relt(1)},
|
||||
testSnap{"2", true, relt(2)},
|
||||
testSnap{"3", false, relt(3)},
|
||||
testSnap{"4", false, relt(4)},
|
||||
testSnap{"5", false, relt(5)},
|
||||
}
|
||||
|
||||
now := relt(6)
|
||||
keep, remove := g.FitEntries(snaps)
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
}
|
||||
|
||||
func TestRespectsKeepCountAll(t *testing.T) {
|
||||
g := gridFromString("1m,-1|1m,1")
|
||||
relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
|
||||
snaps := []Entry{
|
||||
testSnap{"a", true, relt(0)},
|
||||
testSnap{"b", true, relt(-1)},
|
||||
testSnap{"c", true, relt(-2)},
|
||||
testSnap{"d", false, relt(-60)},
|
||||
testSnap{"e", true, relt(-61)},
|
||||
}
|
||||
keep, remove := g.FitEntries(snaps)
|
||||
validateRetentionGridFitEntries(t, relt(61), snaps, keep, remove)
|
||||
}
|
||||
|
||||
func TestComplex(t *testing.T) {
|
||||
|
||||
g := gridFromString("10m,-1|10m|10m,2|1h")
|
||||
|
||||
t.Logf("%#v\n", g)
|
||||
|
||||
now := time.Unix(0, 0)
|
||||
|
||||
snaps := []Entry{
|
||||
// pre-now must always be kept
|
||||
testSnap{"1", true, now.Add(3 * time.Minute)},
|
||||
// 1st interval allows unlimited entries
|
||||
testSnap{"b1", true, now.Add(-6 * time.Minute)},
|
||||
testSnap{"b3", true, now.Add(-8 * time.Minute)},
|
||||
testSnap{"b2", true, now.Add(-9 * time.Minute)},
|
||||
// 2nd interval allows 1 entry
|
||||
testSnap{"a", false, now.Add(-11 * time.Minute)},
|
||||
testSnap{"c", true, now.Add(-19 * time.Minute)},
|
||||
// 3rd interval allows 2 entries
|
||||
testSnap{"foo", true, now.Add(-25 * time.Minute)},
|
||||
testSnap{"bar", true, now.Add(-26 * time.Minute)},
|
||||
// this is at the left edge of the 4th interval
|
||||
testSnap{"border", false, now.Add(-30 * time.Minute)},
|
||||
// right in the 4th interval
|
||||
testSnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
|
||||
// on the right edge of 4th interval => not in it => delete
|
||||
testSnap{"q", false, now.Add(-1*time.Hour - 30*time.Minute)},
|
||||
// older then 4th interval => always delete
|
||||
testSnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)},
|
||||
testSnap{"f", false, now.Add(-2 * time.Hour)},
|
||||
}
|
||||
keep, remove := g.fitEntriesWithNow(now, snaps)
|
||||
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user