replication & pruning: ditch replicated-property, use bookmark as cursor instead

A bookmark with a well-known name is used to track which version was
last successfully received by the receiver.
The createtxg that can be retrieved from the bookmark using `zfs get` is
used to set the Replicated attribute of each snap on the sender:
If the snap's CreateTXG > the cursor's, it is not yet replicated,
otherwise it has been.

There is an optional config option to change the behvior to
`CreateTXG >= the cursor's`, and the implementation defaults to that.

The reason: While things work just fine with `CreateTXG > the cursor's`,
ZFS does not provide size estimates in a `zfs send` dry run
(see acd2418).
However, to enable the use case of keeping the snapshot only around for
the replication, the config flag exists.

zfs/replication_history.go

@@ -1,25 +1,58 @@
 package zfs
 
-const ReplicatedProperty = "zrepl:replicated"
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"strconv"
+)
 
-// May return *DatasetDoesNotExist as an error
-func ZFSGetReplicatedProperty(fs *DatasetPath, v *FilesystemVersion) (replicated bool, err error) {
-	props, err := zfsGet(v.ToAbsPath(fs), []string{ReplicatedProperty})
+const ReplicationCursorBookmarkName = "zrepl_replication_cursor"
+
+// may return nil for both values, indicating there is no cursor
+func ZFSGetReplicationCursor(fs *DatasetPath) (*FilesystemVersion, error) {
+	versions, err := ZFSListFilesystemVersions(fs, nil)
 	if err != nil {
-		return false, err
+		return nil, err
 	}
-	if props.Get(ReplicatedProperty) == "yes" {
-		return true, nil
+	for _, v := range versions {
+		if v.Type == Bookmark && v.Name == ReplicationCursorBookmarkName {
+			return &v, nil
+		}
 	}
-	return false, nil
+	return nil, nil
 }
 
-func ZFSSetReplicatedProperty(fs *DatasetPath, v *FilesystemVersion, replicated bool) error {
-	val := "no"
-	if replicated {
-		val = "yes"
+func ZFSSetReplicationCursor(fs *DatasetPath, snapname string) (guid uint64, err error) {
+	snapPath := fmt.Sprintf("%s@%s", fs.ToString(), snapname)
+	propsSnap, err := zfsGet(snapPath, []string{"createtxg", "guid"})
+	if err != nil {
+		return 0, err
 	}
-	props := NewZFSProperties()
-	props.Set(ReplicatedProperty, val)
-	return zfsSet(v.ToAbsPath(fs), props)
+	snapGuid, err := strconv.ParseUint(propsSnap.Get("guid"), 10, 64)
+	bookmarkPath := fmt.Sprintf("%s#%s", fs.ToString(), ReplicationCursorBookmarkName)
+	propsBookmark, err := zfsGet(bookmarkPath, []string{"createtxg"})
+	_, bookmarkNotExistErr := err.(*DatasetDoesNotExist)
+	if err != nil && !bookmarkNotExistErr {
+		return 0, err
+	}
+	if err == nil {
+		bookmarkTxg, err := strconv.ParseUint(propsBookmark.Get("createtxg"), 10, 64)
+		if err != nil {
+			return 0, errors.Wrap(err, "cannot parse bookmark createtxg")
+		}
+		snapTxg, err := strconv.ParseUint(propsSnap.Get("createtxg"), 10, 64)
+		if err != nil {
+			return 0, errors.Wrap(err, "cannot parse snapshot createtxg")
+		}
+		if snapTxg < bookmarkTxg {
+			return 0, errors.New("replication cursor can only be advanced, not set back")
+		}
+		if err := ZFSDestroy(bookmarkPath); err != nil { // FIXME make safer by using new temporary bookmark, then rename, possible with channel programs
+			return 0, err
+		}
+	}
+	if err := ZFSBookmark(fs, snapname, ReplicationCursorBookmarkName); err != nil {
+		return 0, err
+	}
+	return snapGuid, nil
 }