adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

endpoint: refactor, fix stale holds on initial replication, holds release subcmds

Browse Source 
- endpoint abstractions now share an interface `Abstraction`
- pkg endpoint now has a query facitilty (`ListAbstractions`) which is
  used to find on-disk
    - step holds and bookmarks
    - replication cursors (v1, v2)
    - last-received-holds
- the `zrepl holds list` command consumes endpoint.ListAbstractions
- the new `zrepl holds release-{all,stale}` commands can be used
  to remove abstractions of package endpoint

Co-authored-by: InsanePrawn <insane.prawny@gmail.com>

supersedes #282
fixes #280
fixes #278
This commit is contained in:
Christian Schwarz
2020-03-26 23:43:17 +01:00
parent 44bd354eae
commit f3734ed0d4
28 changed files with 1854 additions and 1168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platformtest/tests/batchRelease.go
-154
View File
@@ -1,154 +0,0 @@
package tests
import (
"fmt"
"github.com/stretchr/testify/require"
"github.com/zrepl/zrepl/platformtest"
"github.com/zrepl/zrepl/zfs"
)
type rollupReleaseExpectTags struct {
Snap string
Holds map[string]bool
}
func rollupReleaseTest(ctx *platformtest.Context, cb func(fs string) []rollupReleaseExpectTags) {
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
DESTROYROOT
CREATEROOT
+ "foo bar"
+ "foo bar@1"
+ "foo bar@2"
+ "foo bar@3"
+ "foo bar@4"
+ "foo bar@5"
+ "foo bar@6"
R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@1"
R zfs hold zrepl_platformtest_2 "${ROOTDS}/foo bar@2"
R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@3"
R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@5"
R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@6"
R zfs bookmark "${ROOTDS}/foo bar@5" "${ROOTDS}/foo bar#5"
`)
fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
expTags := cb(fs)
for _, exp := range expTags {
holds, err := zfs.ZFSHolds(ctx, fs, exp.Snap)
if err != nil {
panic(err)
}
for _, h := range holds {
if e, ok := exp.Holds[h]; !ok || !e {
panic(fmt.Sprintf("tag %q on snap %q not expected", h, exp.Snap))
}
}
}
}
func RollupReleaseIncluding(ctx *platformtest.Context) {
rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
guid5, err := zfs.ZFSGetGUID(fs, "@5")
require.NoError(ctx, err)
err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
require.NoError(ctx, err)
return []rollupReleaseExpectTags{
{"1", map[string]bool{}},
{"2", map[string]bool{"zrepl_platformtest_2": true}},
{"3", map[string]bool{}},
{"4", map[string]bool{}},
{"5", map[string]bool{}},
{"6", map[string]bool{"zrepl_platformtest": true}},
}
})
}
func RollupReleaseExcluding(ctx *platformtest.Context) {
rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
guid5, err := zfs.ZFSGetGUID(fs, "@5")
require.NoError(ctx, err)
err = zfs.ZFSReleaseAllOlderThanGUID(ctx, fs, guid5, "zrepl_platformtest")
require.NoError(ctx, err)
return []rollupReleaseExpectTags{
{"1", map[string]bool{}},
{"2", map[string]bool{"zrepl_platformtest_2": true}},
{"3", map[string]bool{}},
{"4", map[string]bool{}},
{"5", map[string]bool{"zrepl_platformtest": true}},
{"6", map[string]bool{"zrepl_platformtest": true}},
}
})
}
func RollupReleaseMostRecentIsBookmarkWithoutSnapshot(ctx *platformtest.Context) {
rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
guid5, err := zfs.ZFSGetGUID(fs, "#5")
require.NoError(ctx, err)
err = zfs.ZFSRelease(ctx, "zrepl_platformtest", fs+"@5")
require.NoError(ctx, err)
err = zfs.ZFSDestroy(fs + "@5")
require.NoError(ctx, err)
err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
require.NoError(ctx, err)
return []rollupReleaseExpectTags{
{"1", map[string]bool{}},
{"2", map[string]bool{"zrepl_platformtest_2": true}},
{"3", map[string]bool{}},
{"4", map[string]bool{}},
// {"5", map[string]bool{}}, doesn't exist
{"6", map[string]bool{"zrepl_platformtest": true}},
}
})
}
func RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists(ctx *platformtest.Context) {
rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
guid5, err := zfs.ZFSGetGUID(fs, "#5")
require.NoError(ctx, err)
err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
require.NoError(ctx, err)
return []rollupReleaseExpectTags{
{"1", map[string]bool{}},
{"2", map[string]bool{"zrepl_platformtest_2": true}},
{"3", map[string]bool{}},
{"4", map[string]bool{}},
{"5", map[string]bool{}},
{"6", map[string]bool{"zrepl_platformtest": true}},
}
})
}
func RollupReleaseMostRecentDoesntExist(ctx *platformtest.Context) {
rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
const nonexistentGuid = 0 // let's take our chances...
err := zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, nonexistentGuid, "zrepl_platformtest")
require.Error(ctx, err)
require.Contains(ctx, err.Error(), "cannot find snapshot or bookmark with guid 0")
return []rollupReleaseExpectTags{
{"1", map[string]bool{"zrepl_platformtest": true}},
{"2", map[string]bool{"zrepl_platformtest_2": true}},
{"3", map[string]bool{"zrepl_platformtest": true}},
{"4", map[string]bool{"zrepl_platformtest": true}},
{"5", map[string]bool{"zrepl_platformtest": true}},
{"6", map[string]bool{"zrepl_platformtest": true}},
}
})
}
platformtest/tests/helpers.go
+18 -5
View File
@@ -25,6 +25,14 @@ func sendArgVersion(fs, relName string) zfs.ZFSSendArgVersion {
}
}
func fsversion(fs, relname string) zfs.FilesystemVersion {
v, err := zfs.ZFSGetFilesystemVersion(fs + relname)
if err != nil {
panic(err)
}
return v
}
func mustDatasetPath(fs string) *zfs.DatasetPath {
p, err := zfs.NewDatasetPath(fs)
if err != nil {
@@ -47,8 +55,8 @@ func mustSnapshot(snap string) {
}
}
func mustGetProps(entity string) zfs.ZFSPropCreateTxgAndGuidProps {
props, err := zfs.ZFSGetCreateTXGAndGuid(entity)
func mustGetFilesystemVersion(snapOrBookmark string) zfs.FilesystemVersion {
props, err := zfs.ZFSGetFilesystemVersion(snapOrBookmark)
check(err)
return props
}
@@ -78,7 +86,7 @@ type dummySnapshotSituation struct {
}
type resumeSituation struct {
sendArgs zfs.ZFSSendArgs
sendArgs zfs.ZFSSendArgsUnvalidated
recvOpts zfs.RecvOptions
sendErr, recvErr error
recvErrDecoded *zfs.RecvFailedWithResumeTokenErr
@@ -107,7 +115,7 @@ func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation
return situation
}
func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgs, recvOptions zfs.RecvOptions) *resumeSituation {
func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgsUnvalidated, recvOptions zfs.RecvOptions) *resumeSituation {
situation := &resumeSituation{}
@@ -115,8 +123,13 @@ func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation,
situation.recvOpts = recvOptions
require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointless otherwise")
require.Equal(ctx, sendArgs.FS, src.sendFS)
sendArgsValidated, err := sendArgs.Validate(ctx)
situation.sendErr = err
if err != nil {
return situation
}
copier, err := zfs.ZFSSend(ctx, sendArgs)
copier, err := zfs.ZFSSend(ctx, sendArgsValidated)
situation.sendErr = err
if err != nil {
return situation
platformtest/tests/idempotentHold.go
+1 -11
View File
@@ -22,7 +22,7 @@ func IdempotentHold(ctx *platformtest.Context) {
`)
fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
v1 := sendArgVersion(fs, "@1")
v1 := fsversion(fs, "@1")
tag := "zrepl_platformtest"
err := zfs.ZFSHold(ctx, fs, v1, tag)
@@ -34,14 +34,4 @@ func IdempotentHold(ctx *platformtest.Context) {
if err != nil {
panic(err)
}
vnonexistent := zfs.ZFSSendArgVersion{
RelName: "@nonexistent",
GUID: 0xbadf00d,
}
err = zfs.ZFSHold(ctx, fs, vnonexistent, tag)
if err == nil {
panic("still expecting error for nonexistent snapshot")
}
}
platformtest/tests/replicationCursor.go
+6 -6
View File
@@ -31,7 +31,7 @@ func ReplicationCursor(ctx *platformtest.Context) {
}
fs := ds.ToString()
snap := sendArgVersion(fs, "@1 with space")
snap := fsversion(fs, "@1 with space")
destroyed, err := endpoint.MoveReplicationCursor(ctx, fs, &snap, jobid)
if err != nil {
@@ -39,7 +39,7 @@ func ReplicationCursor(ctx *platformtest.Context) {
}
assert.Empty(ctx, destroyed)
snapProps, err := zfs.ZFSGetCreateTXGAndGuid(snap.FullPath(fs))
snapProps, err := zfs.ZFSGetFilesystemVersion(snap.FullPath(fs))
if err != nil {
panic(err)
}
@@ -56,13 +56,13 @@ func ReplicationCursor(ctx *platformtest.Context) {
}
// try moving
cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.GUID, jobid)
cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.Guid, jobid)
require.NoError(ctx, err)
snap2 := sendArgVersion(fs, "@2 with space")
snap2 := fsversion(fs, "@2 with space")
destroyed, err = endpoint.MoveReplicationCursor(ctx, fs, &snap2, jobid)
require.NoError(ctx, err)
require.Equal(ctx, 1, len(destroyed))
require.Equal(ctx, zfs.Bookmark, destroyed[0].Type)
require.Equal(ctx, cursor1BookmarkName, destroyed[0].Name)
require.Equal(ctx, endpoint.AbstractionReplicationCursorBookmarkV2, destroyed[0].GetType())
require.Equal(ctx, cursor1BookmarkName, destroyed[0].GetName())
}
platformtest/tests/resumableRecvAndTokenHandling.go
+1 -1
View File
@@ -25,7 +25,7 @@ func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
src := makeDummyDataSnapshots(ctx, sendFS)
s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgs{
s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: false},
platformtest/tests/sendArgsValidation.go
+19 -29
View File
@@ -23,9 +23,9 @@ func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformt
`)
fs := fmt.Sprintf("%s/send er", ctx.RootDataset)
props := mustGetProps(fs + "@a snap")
props := mustGetFilesystemVersion(fs + "@a snap")
sendArgs := zfs.ZFSSendArgs{
sendArgs, err := zfs.ZFSSendArgsUnvalidated{
FS: fs,
To: &zfs.ZFSSendArgVersion{
RelName: "@a snap",
@@ -33,10 +33,15 @@ func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformt
},
Encrypted: &zfs.NilBool{B: true},
ResumeToken: "",
}
stream, err := zfs.ZFSSend(ctx, sendArgs)
}.Validate(ctx)
var stream *zfs.ReadCloserCopier
if err == nil {
defer stream.Close()
stream, err = zfs.ZFSSend(ctx, sendArgs) // no shadow
if err == nil {
defer stream.Close()
}
// fallthrough
}
if expectNotSupportedErr {
@@ -76,7 +81,7 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
src := makeDummyDataSnapshots(ctx, sendFS)
unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgs{
unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: false}, // !
@@ -85,7 +90,7 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
SavePartialRecvState: true,
})
encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgs{
encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS,
To: src.snapA,
Encrypted: &zfs.NilBool{B: true}, // !
@@ -97,16 +102,10 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
// threat model: use of a crafted resume token that requests an unencrypted send
// but send args require encrypted send
{
var maliciousSend zfs.ZFSSendArgs = encS.sendArgs
var maliciousSend zfs.ZFSSendArgsUnvalidated = encS.sendArgs
maliciousSend.ResumeToken = unencS.recvErrDecoded.ResumeTokenRaw
stream, err := zfs.ZFSSend(ctx, maliciousSend)
if err == nil {
defer stream.Close()
}
require.Nil(ctx, stream)
require.Error(ctx, err)
ctx.Logf("send err: %T %s", err, err)
_, err := maliciousSend.Validate(ctx)
validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
require.True(ctx, ok)
require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
@@ -120,14 +119,10 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
// threat model: use of a crafted resume token that requests an encrypted send
// but send args require unencrypted send
{
var maliciousSend zfs.ZFSSendArgs = unencS.sendArgs
var maliciousSend zfs.ZFSSendArgsUnvalidated = unencS.sendArgs
maliciousSend.ResumeToken = encS.recvErrDecoded.ResumeTokenRaw
stream, err := zfs.ZFSSend(ctx, maliciousSend)
if err == nil {
defer stream.Close()
}
require.Nil(ctx, stream)
_, err := maliciousSend.Validate(ctx)
require.Error(ctx, err)
ctx.Logf("send err: %T %s", err, err)
validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
@@ -169,7 +164,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
src1 := makeDummyDataSnapshots(ctx, sendFS1)
src2 := makeDummyDataSnapshots(ctx, sendFS2)
rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgs{
rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{
FS: sendFS1,
To: src1.snapA,
Encrypted: &zfs.NilBool{B: false},
@@ -180,7 +175,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
// threat model: forged resume token tries to steal a full send of snapA on fs2 by
// presenting a resume token for full send of snapA on fs1
var maliciousSend zfs.ZFSSendArgs = zfs.ZFSSendArgs{
var maliciousSend zfs.ZFSSendArgsUnvalidated = zfs.ZFSSendArgsUnvalidated{
FS: sendFS2,
To: &zfs.ZFSSendArgVersion{
RelName: src2.snapA.RelName,
@@ -189,12 +184,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
Encrypted: &zfs.NilBool{B: false},
ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
}
stream, err := zfs.ZFSSend(ctx, maliciousSend)
if err == nil {
defer stream.Close()
}
require.Nil(ctx, stream)
_, err = maliciousSend.Validate(ctx)
require.Error(ctx, err)
ctx.Logf("send err: %T %s", err, err)
validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
platformtest/tests/tests.go
-5
View File
@@ -18,11 +18,6 @@ var Cases = []Case{
UndestroyableSnapshotParsing,
GetNonexistent,
ReplicationCursor,
RollupReleaseIncluding,
RollupReleaseExcluding,
RollupReleaseMostRecentIsBookmarkWithoutSnapshot,
RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists,
RollupReleaseMostRecentDoesntExist,
IdempotentHold,
IdempotentBookmark,
IdempotentDestroy,