move implementation to internal/ directory (#828)

2024-10-18 19:21:17 +02:00
parent b9b9ad10cf
commit 908807bd59
360 changed files with 507 additions and 507 deletions
@@ -0,0 +1,142 @@
+package retentiongrid
+
+import (
+	"sort"
+	"time"
+)
+
+type Interval interface {
+	Length() time.Duration
+	KeepCount() int
+}
+
+const RetentionGridKeepCountAll int = -1
+
+type Grid struct {
+	intervals []Interval
+}
+
+type Entry interface {
+	Date() time.Time
+}
+
+func NewGrid(l []Interval) *Grid {
+	if len(l) == 0 {
+		panic("must specify at least one interval")
+	}
+	// TODO Maybe check for ascending interval lengths here, although the algorithm
+	// 		itself doesn't care about that.
+	return &Grid{l}
+}
+
+func (g Grid) FitEntries(entries []Entry) (keep, remove []Entry) {
+
+	if len(entries) == 0 {
+		return
+	}
+
+	// determine 'now' based on youngest snapshot
+	// => sort youngest-to-oldest
+	sort.SliceStable(entries, func(i, j int) bool {
+		return entries[i].Date().After(entries[j].Date())
+	})
+	now := entries[0].Date()
+
+	return g.fitEntriesWithNow(now, entries)
+}
+
+type bucket struct {
+	keepCount     int
+	youngerThan   time.Time
+	olderThanOrEq time.Time
+	entries       []Entry
+}
+
+func makeBucketFromInterval(olderThanOrEq time.Time, i Interval) bucket {
+	var b bucket
+	kc := i.KeepCount()
+	if kc == 0 {
+		panic("keep count 0 is not allowed")
+	}
+	if (kc < 0) && kc != RetentionGridKeepCountAll {
+		panic("negative keep counts are not allowed")
+	}
+	b.keepCount = kc
+	b.olderThanOrEq = olderThanOrEq
+	b.youngerThan = b.olderThanOrEq.Add(-i.Length())
+	return b
+}
+
+func (b *bucket) Contains(e Entry) bool {
+	d := e.Date()
+	olderThan := d.Before(b.olderThanOrEq)
+	eq := d.Equal(b.olderThanOrEq)
+	youngerThan := d.After(b.youngerThan)
+	return (olderThan || eq) && youngerThan
+}
+
+func (b *bucket) AddIfContains(e Entry) (added bool) {
+	added = b.Contains(e)
+	if added {
+		b.entries = append(b.entries, e)
+	}
+	return
+}
+
+func (b *bucket) RemoveYoungerSnapsExceedingKeepCount() (removed []Entry) {
+
+	if b.keepCount == RetentionGridKeepCountAll {
+		return nil
+	}
+
+	removeCount := len(b.entries) - b.keepCount
+	if removeCount <= 0 {
+		return nil
+	}
+
+	// sort youngest-to-oldest
+	sort.SliceStable(b.entries, func(i, j int) bool {
+		return b.entries[i].Date().After(b.entries[j].Date())
+	})
+
+	return b.entries[:removeCount]
+}
+
+func (g Grid) fitEntriesWithNow(now time.Time, entries []Entry) (keep, remove []Entry) {
+
+	buckets := make([]bucket, len(g.intervals))
+
+	buckets[0] = makeBucketFromInterval(now, g.intervals[0])
+	for i := 1; i < len(g.intervals); i++ {
+		buckets[i] = makeBucketFromInterval(buckets[i-1].youngerThan, g.intervals[i])
+	}
+
+	keep = make([]Entry, 0)
+	remove = make([]Entry, 0)
+
+assignEntriesToBuckets:
+	for ei := 0; ei < len(entries); ei++ {
+		e := entries[ei]
+		// unconditionally keep entries that are in the future
+		if now.Before(e.Date()) {
+			keep = append(keep, e)
+			continue assignEntriesToBuckets
+		}
+		// add to matching bucket, if any
+		for bi := range buckets {
+			if buckets[bi].AddIfContains(e) {
+				continue assignEntriesToBuckets
+			}
+		}
+		// unconditionally remove entries older than the oldest bucket
+		remove = append(remove, e)
+	}
+
+	// now apply the `KeepCount` per bucket
+	for _, b := range buckets {
+		destroy := b.RemoveYoungerSnapsExceedingKeepCount()
+		remove = append(remove, destroy...)
+		keep = append(keep, b.entries...)
+	}
+	return
+}
@@ -0,0 +1,185 @@
+package retentiongrid
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type testInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *testInterval) Length() time.Duration { return i.length }
+func (i *testInterval) KeepCount() int        { return i.keepCount }
+
+func gridFromString(gs string) (g *Grid) {
+	sintervals := strings.Split(gs, "|")
+	intervals := make([]Interval, len(sintervals))
+	for idx, i := range sintervals {
+		comps := strings.SplitN(i, ",", 2)
+		var durationStr, numSnapsStr string
+		durationStr = comps[0]
+		if len(comps) == 1 {
+			numSnapsStr = "1"
+		} else {
+			numSnapsStr = comps[1]
+		}
+
+		var err error
+		var interval testInterval
+
+		if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil {
+			panic(err)
+		}
+		if interval.length, err = time.ParseDuration(durationStr); err != nil {
+			panic(err)
+		}
+
+		intervals[idx] = &interval
+	}
+	return NewGrid(intervals)
+}
+
+type testSnap struct {
+	Name       string
+	ShouldKeep bool
+	date       time.Time
+}
+
+func (ds testSnap) Date() time.Time { return ds.date }
+
+func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) {
+
+	snapDescr := func(d testSnap) string {
+		return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now))
+	}
+
+	t.Logf("keep list:\n")
+	for k := range keep {
+		t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
+	}
+	t.Logf("remove list:\n")
+	for k := range remove {
+		t.Logf("\t%s\n", snapDescr(remove[k].(testSnap)))
+	}
+
+	t.Logf("\n\n")
+
+	for _, s := range input {
+		d := s.(testSnap)
+		descr := snapDescr(d)
+		t.Logf("testing %s\n", descr)
+		if d.ShouldKeep {
+			assert.Contains(t, keep, d, "expecting %s to be kept", descr)
+		} else {
+			assert.Contains(t, remove, d, "expecting %s to be removed", descr)
+		}
+	}
+
+	t.Logf("resulting list:\n")
+	for k := range keep {
+		t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
+	}
+}
+
+func TestEmptyInput(t *testing.T) {
+	g := gridFromString("10m|10m|10m|1h")
+	keep, remove := g.FitEntries([]Entry{})
+	assert.Empty(t, keep)
+	assert.Empty(t, remove)
+}
+
+func TestIntervalBoundariesAndAlignment(t *testing.T) {
+	g := gridFromString("10m|10m|10m")
+
+	t.Logf("%#v\n", g)
+
+	now := time.Unix(0, 0)
+
+	snaps := []Entry{
+		testSnap{"0", true, now.Add(1 * time.Minute)},    // before now => keep unconditionally
+		testSnap{"1", true, now},                         // 1st interval left edge => inclusive
+		testSnap{"2", true, now.Add(-10 * time.Minute)},  // 2nd interval left edge => inclusive
+		testSnap{"3", true, now.Add(-20 * time.Minute)},  // 3rd interval left edge => inclusuive
+		testSnap{"4", false, now.Add(-30 * time.Minute)}, // 3rd interval right edge => excludive
+		testSnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval => remove unconditionally
+	}
+
+	keep, remove := g.fitEntriesWithNow(now, snaps)
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+}
+
+func TestKeepsOldestSnapsInABucket(t *testing.T) {
+	g := gridFromString("1m,2")
+
+	relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
+
+	snaps := []Entry{
+		testSnap{"1", true, relt(1)},
+		testSnap{"2", true, relt(2)},
+		testSnap{"3", false, relt(3)},
+		testSnap{"4", false, relt(4)},
+		testSnap{"5", false, relt(5)},
+	}
+
+	now := relt(6)
+	keep, remove := g.FitEntries(snaps)
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+}
+
+func TestRespectsKeepCountAll(t *testing.T) {
+	g := gridFromString("1m,-1|1m,1")
+	relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
+	snaps := []Entry{
+		testSnap{"a", true, relt(0)},
+		testSnap{"b", true, relt(-1)},
+		testSnap{"c", true, relt(-2)},
+		testSnap{"d", false, relt(-60)},
+		testSnap{"e", true, relt(-61)},
+	}
+	keep, remove := g.FitEntries(snaps)
+	validateRetentionGridFitEntries(t, relt(61), snaps, keep, remove)
+}
+
+func TestComplex(t *testing.T) {
+
+	g := gridFromString("10m,-1|10m|10m,2|1h")
+
+	t.Logf("%#v\n", g)
+
+	now := time.Unix(0, 0)
+
+	snaps := []Entry{
+		// pre-now must always be kept
+		testSnap{"1", true, now.Add(3 * time.Minute)},
+		// 1st interval allows unlimited entries
+		testSnap{"b1", true, now.Add(-6 * time.Minute)},
+		testSnap{"b3", true, now.Add(-8 * time.Minute)},
+		testSnap{"b2", true, now.Add(-9 * time.Minute)},
+		// 2nd interval allows 1 entry
+		testSnap{"a", false, now.Add(-11 * time.Minute)},
+		testSnap{"c", true, now.Add(-19 * time.Minute)},
+		// 3rd interval allows 2 entries
+		testSnap{"foo", true, now.Add(-25 * time.Minute)},
+		testSnap{"bar", true, now.Add(-26 * time.Minute)},
+		// this is at the left edge of the 4th interval
+		testSnap{"border", false, now.Add(-30 * time.Minute)},
+		// right in the 4th interval
+		testSnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
+		// on the right edge of 4th interval => not in it => delete
+		testSnap{"q", false, now.Add(-1*time.Hour - 30*time.Minute)},
+		// older then 4th interval => always delete
+		testSnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)},
+		testSnap{"f", false, now.Add(-2 * time.Hour)},
+	}
+	keep, remove := g.fitEntriesWithNow(now, snaps)
+
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+
+}