WIP

From https://github.com/zrepl/zrepl/issues/691

The last_n prune rule keeps everything, regardless of if it matches the
regex or not, if there are less than count snapshot. The expectation
would be to never keep non-regex snapshots, regardless of number.

.circleci/config.yml

@@ -58,19 +58,26 @@ commands:
             git config --global user.email "zreplbot@cschwarz.com"
             git config --global user.name "zrepl-github-io-ci"
 
-      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      # if we're pushing, we need to add the deploy key
-      - run: ssh-add -D
+      # which is stored as "Additional SSH Keys" in the CircleCI project settings.
-      # the default circleci ssh config only additional ssh keys for Host !github.com
+      # We can't use the CircleCI-manage deploy key because we're pushing
-      - run:
+      # to a different repo than the one we're building.
-          command: |
+      - when:
-            cat > ~/.ssh/config \<<EOF
+          condition: << parameters.push >>
-            Host *
+          steps:
-              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
-            EOF
+            - run: ssh-add -D
-      - add_ssh_keys:
+            # the default circleci ssh config only additional ssh keys for Host !github.com
-          fingerprints:
+            - run:
-            # deploy key for zrepl.github.io
+                command: |
-            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
+                  cat > ~/.ssh/config \<<EOF
+                  Host *
+                    IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+                  EOF
+            - add_ssh_keys:
+                fingerprints:
+                  # deploy key for zrepl.github.io
+                  - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
 
       # caller must install-docdep
       - when:
@@ -283,6 +290,7 @@ jobs:
     steps:
       - attach_workspace:
           at: .
+      - run: make wrapup-and-checksum
       - store_artifacts:
           path: artifacts
 

.circleci/download_artifacts.py

@@ -0,0 +1,84 @@
+import argparse
+from pathlib import Path
+import re
+
+import requests
+
+import time
+
+import os
+import argparse
+from pathlib import Path
+
+
+circle_token = os.environ.get('CIRCLE_TOKEN')
+if not circle_token:
+    raise ValueError('CIRCLE_TOKEN environment variable not set')
+
+parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
+parser.add_argument('build_num', type=str, help='Build number')
+parser.add_argument('dst', type=Path, help='Destination directory')
+parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
+parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
+
+args = parser.parse_args()
+
+res = requests.get(
+    f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
+    headers={
+        "Circle-Token": circle_token,
+    },
+)
+res.raise_for_status()
+
+# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
+# [ {
+#   "path" : "raw-test-output/go-test-report.xml",
+#   "pretty_path" : "raw-test-output/go-test-report.xml",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
+# }, {
+#   "path" : "raw-test-output/go-test.out",
+#   "pretty_path" : "raw-test-output/go-test.out",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
+# } ]
+res = res.json()
+
+for artifact in res:
+    if not artifact["pretty_path"].startswith(args.prefix):
+        continue
+    if not re.match(args.match, artifact["pretty_path"]):
+        continue
+    stripped = artifact["pretty_path"][len(args.prefix):]
+    print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
+    artifact_rel = Path(stripped)
+    artifact_dst = args.dst / artifact_rel
+    artifact_dst.parent.mkdir(parents=True, exist_ok=True)
+
+    res = requests.get(
+        artifact["url"],
+        headers={
+            "Circle-Token": circle_token,
+        },
+        stream=True,
+    )
+    res.raise_for_status()
+
+    total_size = int(res.headers.get("Content-Length", 0))
+    block_size = 128 * 1024
+    with open(artifact_dst, "wb") as f:
+        progress = 0
+        start_time = time.time()
+        for chunk in res.iter_content(chunk_size=block_size):
+            f.write(chunk)
+            progress += len(chunk)
+            percent = progress / total_size * 100
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= 5:
+                print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
+                start_time = time.time()
+        print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
+    print("Download complete!")
+
+print("All files downloaded")

Makefile

@@ -1,5 +1,5 @@
 .PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.PHONY: release bins-all release-noarch
+.PHONY: release release-noarch
 .DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
@@ -14,13 +14,15 @@ ifndef _ZREPL_VERSION
     endif
 endif
 
+ZREPL_PACKAGE_RELEASE := 1
+
 GO := go
 GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
 GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
 GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
 GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
 GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
-GO_ENV_VARS := GO111MODULE=on
+GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
 GO_EXTRA_BUILDFLAGS :=
@@ -50,21 +52,20 @@ printvars:
 release: clean
 	# no cross-platform support for target test
 	$(MAKE) test-go
-	$(MAKE) bins-all
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
 	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(MAKE) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
 release-docker: $(ARTIFACTDIR)
 	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
 	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
 	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
 		zrepl_release \
-		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make release \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
 debs-docker:
 	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
@@ -81,9 +82,14 @@ rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion
 	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
 	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
 	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
-	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
-	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+		mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
-	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+	done
+	sed \
+		-e "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+		-e "s/^Release:.*/Release:          $(ZREPL_PACKAGE_RELEASE)/g" \
+	    packaging/rpm/zrepl.spec \
+		>  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
 
 	# see /usr/lib/rpm/platform
 ifeq ($(GOARCH),amd64)
@@ -110,13 +116,16 @@ rpm-docker:
 	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
 		zrepl_rpm_pkg \
-		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make rpm \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
+
 
 deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
 
 	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
 	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
-	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
 		export VERSION="$${VERSION#v}"; \
 		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
 
@@ -142,9 +151,11 @@ deb-docker:
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
 		--ulimit nofile=1024:1024 \
 		zrepl_debian_pkg \
-		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make deb \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
-# expects `release` target to have run before
+# expects `release`, `deb` & `rpm` targets to have run before
 NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
 wrapup-and-checksum:
 	rm -f $(NOARCH_TARBALL)
@@ -161,7 +172,7 @@ wrapup-and-checksum:
 		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp -l $(ARTIFACTDIR)/zrepl-* \
+	cp -l $(ARTIFACTDIR)/zrepl* \
 		$(ARTIFACTDIR)/platformtest-* \
 		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
@@ -181,39 +192,42 @@ check-git-clean:
 
 tag-release:
 	test -n "$(ZREPL_TAG_VERSION)" || exit 1
-	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+	git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
 
 sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+	gpg -u '328A6627FA98061D!' \
 		--armor \
 		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
 
-##################### BINARIES #####################
+download-circleci-release:
-.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+	rm -rf "$(ARTIFACTDIR)"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
 
-BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+##################### MULTI-ARCH HELPERS #####################
-GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+
-bins-all:
+_run_make_foreach_target_tuple:
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+		echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
+		exit 1; \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
+	fi
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm	  GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm64
-ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm     GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=386
-else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin    GOARCH=amd64
-	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris   GOARCH=amd64
-else
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos   GOARCH=amd64
-	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+
-endif
+##################### REGULAR TARGETS #####################
+.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
 
 lint:
 	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...

config/config.go

@@ -121,6 +121,7 @@ type BandwidthLimit struct {
 }
 
 type Replication struct {
+	Triggers    []*ReplicationTriggerEnum
 	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
 	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
 }
@@ -135,6 +136,32 @@ type ReplicationOptionsConcurrency struct {
 	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
 }
 
+type ReplicationTriggerEnum struct {
+	Ret interface{}
+}
+
+func (t *ReplicationTriggerEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"manual":   &ReplicationTriggerManual{},
+		"periodic": &ReplicationTriggerPeriodic{},
+	})
+	return
+}
+
+type ReplicationTriggerManual struct {
+	Type string `yaml:"type"`
+}
+
+type ReplicationTriggerPeriodic struct {
+	Type     string            `yaml:"type"`
+	Interval *PositiveDuration `yaml:"interval"`
+}
+
+type ReplicationTriggerCron struct {
+	Type string   `yaml:"type"`
+	Cron CronSpec `yaml:"cron"`
+}
+
 type PropertyRecvOptions struct {
 	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
 	Override map[zfsprop.Property]string `yaml:"override,optional"`
@@ -157,7 +184,6 @@ func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
 type PullJob struct {
 	ActiveJob `yaml:",inline"`
 	RootFS    string                   `yaml:"root_fs"`
-	Interval  PositiveDurationOrManual `yaml:"interval"`
 	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
 }
 

daemon/job/active.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job/reset"
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
@@ -44,6 +45,8 @@ type ActiveSide struct {
 	promReplicationErrors prometheus.Gauge
 	promLastSuccessful    prometheus.Gauge
 
+	triggers *trigger.Triggers
+
 	tasksMtx sync.Mutex
 	tasks    activeSideTasks
 }
@@ -90,7 +93,7 @@ type activeMode interface {
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
 	PlannerPolicy() logic.PlannerPolicy
-	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
+	RunPeriodic(ctx context.Context, wakeReplication *trigger.Manual)
 	SnapperReport() *snapper.Report
 	ResetConnectBackoff()
 }
@@ -132,8 +135,8 @@ func (m *modePush) Type() Type { return TypePush }
 
 func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
 
-func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+func (m *modePush) RunPeriodic(ctx context.Context, trigger *trigger.Manual) {
-	m.snapper.Run(ctx, wakeUpCommon)
+	m.snapper.Run(ctx, trigger)
 }
 
 func (m *modePush) SnapperReport() *snapper.Report {
@@ -221,7 +224,7 @@ func (*modePull) Type() Type { return TypePull }
 
 func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
 
-func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+func (m *modePull) RunPeriodic(ctx context.Context, wakeReplication *trigger.Manual) {
 	if m.interval.Manual {
 		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
 		// "waiting for wakeups" is printed in common ActiveSide.do
@@ -232,14 +235,7 @@ func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
 	for {
 		select {
 		case <-t.C:
-			select {
+			wakeReplication.Fire()
-			case wakeUpCommon <- struct{}{}:
-			default:
-				GetLogger(ctx).
-					WithField("pull_interval", m.interval).
-					Warn("pull job took longer than pull interval")
-				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
-			}
 		case <-ctx.Done():
 			return
 		}
@@ -370,6 +366,11 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}, p
 		return nil, errors.Wrap(err, "cannot build replication driver config")
 	}
 
+	j.triggers, err = trigger.FromConfig(in.Replication.Triggers)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build triggers")
+	}
+
 	return j, nil
 }
 
@@ -444,12 +445,17 @@ func (j *ActiveSide) Run(ctx context.Context) {
 
 	defer log.Info("job exiting")
 
-	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
+
+	periodCtx, endTask := trace.WithTask(ctx, "periodic")
+	defer endTask()
+	go j.mode.RunPeriodic(periodCtx, periodicTrigger)
+
+	wakeupTrigger := wakeup.Trigger(ctx)
+
+	triggered, endTask := j.triggers.Spawn(ctx, []*trigger.Trigger{periodicTrigger, wakeupTrigger})
 	defer endTask()
-	go j.mode.RunPeriodic(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -459,10 +465,15 @@ outer:
 		case <-ctx.Done():
 			log.WithError(ctx.Err()).Info("context")
 			break outer
-
+		case trigger := <-triggered:
-		case <-wakeup.Wait(ctx):
+			log :=
-			j.mode.ResetConnectBackoff()
+				log.WithField("trigger_id", trigger.ID())
-		case <-periodicDone:
+			log.Info("triggered")
+			switch trigger {
+			case wakeupTrigger:
+				log.Info("trigger is wakeup command, resetting connection backoff")
+				j.mode.ResetConnectBackoff()
+			}
 		}
 		invocationCount++
 		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))

daemon/job/build_jobs.go

@@ -24,21 +24,6 @@ func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, erro
 		js[i] = j
 	}
 
-	// receiving-side root filesystems must not overlap
-	{
-		rfss := make([]string, 0, len(js))
-		for _, j := range js {
-			jrfs, ok := j.OwnedDatasetSubtreeRoot()
-			if !ok {
-				continue
-			}
-			rfss = append(rfss, jrfs.ToString())
-		}
-		if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
-			return nil, err
-		}
-	}
-
 	return js, nil
 }
 

daemon/job/snapjob.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
@@ -104,12 +105,18 @@ func (j *SnapJob) Run(ctx context.Context) {
 
 	defer log.Info("job exiting")
 
-	periodicDone := make(chan struct{})
+	wakeupTrigger := wakeup.Trigger(ctx)
+
+	snapshottingTrigger := trigger.New("periodic")
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
 	defer endTask()
-	go j.snapper.Run(periodicCtx, periodicDone)
+	go j.snapper.Run(periodicCtx, snapshottingTrigger)
+
+	triggers := trigger.Empty()
+	triggered, endTask := triggers.Spawn(ctx, []trigger.Trigger{snapshottingTrigger, wakeupTrigger})
+	defer endTask()
 
 	invocationCount := 0
 outer:
@@ -119,9 +126,7 @@ outer:
 		case <-ctx.Done():
 			log.WithError(ctx.Err()).Info("context")
 			break outer
-
+		case <-triggered:
-		case <-wakeup.Wait(ctx):
-		case <-periodicDone:
 		}
 		invocationCount++
 

daemon/job/trigger/cron.go

@@ -0,0 +1,23 @@
+package trigger
+
+import (
+	"context"
+
+	"github.com/robfig/cron/v3"
+)
+
+type Cron struct {
+	spec cron.Schedule
+}
+
+var _ Trigger = &Cron{}
+
+func NewCron(spec cron.Schedule) *Cron {
+	return &Cron{spec: spec}
+}
+
+func (t *Cron) ID() string { return "cron" }
+
+func (t *Cron) run(ctx context.Context, signal chan<- struct{}) {
+	panic("unimpl: extract from cron snapper")
+}

daemon/job/trigger/from_config.go

@@ -0,0 +1,30 @@
+package trigger
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+func FromConfig(in []*config.ReplicationTriggerEnum) (*Triggers, error) {
+	triggers := make([]Trigger, len(in))
+	for i, e := range in {
+		var t Trigger = nil
+		switch te := e.Ret.(type) {
+		case *config.ReplicationTriggerManual:
+			// not a trigger
+			t = NewManual("manual")
+		case *config.ReplicationTriggerPeriodic:
+			t = NewPeriodic(te.Interval.Duration())
+		case *config.ReplicationTriggerCron:
+			t = NewCron(te.Cron.Schedule)
+		default:
+			return nil, fmt.Errorf("unknown trigger type %T", te)
+		}
+		triggers[i] = t
+	}
+	return &Triggers{
+		spawned:  false,
+		triggers: triggers,
+	}, nil
+}

daemon/job/trigger/logger.go

@@ -0,0 +1,12 @@
+package trigger
+
+import (
+	"context"
+
+	"github.com/zrepl/zrepl/logger"
+)
+
+
+func getLogger(ctx context.Context) logger.Logger {
+	panic("unimpl")
+}

daemon/job/trigger/manual.go

@@ -0,0 +1,33 @@
+package trigger
+
+import "context"
+
+type Manual struct {
+	id     string
+	signal chan<- struct{}
+}
+
+var _ Trigger = &Manual{}
+
+func NewManual(id string) *Manual {
+	return &Manual{
+		id:     id,
+		signal: nil,
+	}
+}
+
+func (t *Manual) ID() string {
+	return t.id
+}
+
+func (t *Manual) run(ctx context.Context, signal chan<- struct{}) {
+	if t.signal != nil {
+		panic("run must only be called once")
+	}
+	t.signal = signal
+}
+
+// Panics if called before the trigger has been spanwed as part of a `Triggers`.
+func (t *Manual) Fire() {
+	t.signal <- struct{}{}
+}

daemon/job/trigger/periodic.go

@@ -0,0 +1,33 @@
+package trigger
+
+import (
+	"context"
+	"time"
+)
+
+type Periodic struct {
+	interval time.Duration
+}
+
+var _ Trigger = &Periodic{}
+
+func NewPeriodic(interval time.Duration) *Periodic {
+	return &Periodic{
+		interval: interval,
+	}
+}
+
+func (p *Periodic) ID() string { return "periodic" }
+
+func (p *Periodic) run(ctx context.Context, signal chan<- struct{}) {
+	t := time.NewTicker(p.interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-t.C:
+			signal <- struct{}{}
+		case <-ctx.Done():
+			return
+		}
+	}
+}

daemon/job/trigger/trigger.go

@@ -0,0 +1,91 @@
+package trigger
+
+import (
+	"context"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+)
+
+type Triggers struct {
+	spawned  bool
+	triggers []Trigger
+}
+
+type Trigger interface {
+	ID() string
+	run(context.Context, chan<- struct{})
+}
+
+func Empty() *Triggers {
+	return &Triggers{
+		spawned:  false,
+		triggers: nil,
+	}
+}
+
+func (t *Triggers) Spawn(ctx context.Context, additionalTriggers []Trigger) (chan Trigger, trace.DoneFunc) {
+	if t.spawned {
+		panic("must only spawn once")
+	}
+	t.spawned = true
+	t.triggers = append(t.triggers, additionalTriggers...)
+	sink := make(chan Trigger)
+	endTask := t.spawn(ctx, sink)
+	return sink, endTask
+}
+
+type triggering struct {
+	trigger Trigger
+	handled chan struct{}
+}
+
+func (t *Triggers) spawn(ctx context.Context, sink chan Trigger) trace.DoneFunc {
+	ctx, endTask := trace.WithTask(ctx, "triggers")
+	ctx, add, wait := trace.WithTaskGroup(ctx, "trigger-tasks")
+	triggered := make(chan triggering, len(t.triggers))
+	for _, t := range t.triggers {
+		t := t
+		signal := make(chan struct{})
+		go add(func(ctx context.Context) {
+			t.run(ctx, signal)
+		})
+		go func() {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-signal:
+					handled := make(chan struct{})
+					select {
+					case triggered <- triggering{trigger: t, handled: handled}:
+					default:
+						panic("this funtion ensures that there's always room in the channel")
+					}
+					select {
+					case <-handled:
+					case <-ctx.Done():
+						return
+					}
+				}
+			}
+		}()
+	}
+	go func() {
+		defer wait()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case triggering := <-triggered:
+				select {
+				case sink <- triggering.trigger:
+				default:
+					getLogger(ctx).
+						WithField("trigger_id", triggering.trigger.ID()).
+						Warn("dropping triggering because job is busy")
+				}
+				close(triggering.handled)
+			}
+		}
+	}()
+	return endTask
+}

daemon/job/wakeup/wakeup.go

@@ -3,6 +3,8 @@ package wakeup
 import (
 	"context"
 	"errors"
+
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 )
 
 type contextKey int
@@ -17,6 +19,10 @@ func Wait(ctx context.Context) <-chan struct{} {
 	return wc
 }
 
+func Trigger(ctx context.Context) trigger.Trigger {
+	panic("unimpl")
+}
+
 type Func func() error
 
 var AlreadyWokenUp = errors.New("already woken up")

daemon/logging/build_logging.go

@@ -63,6 +63,7 @@ type Subsystem string
 const (
 	SubsysMeta         Subsystem = "meta"
 	SubsysJob          Subsystem = "job"
+	SubsysTrigger      Subsystem = "trigger"
 	SubsysReplication  Subsystem = "repl"
 	SubsysEndpoint     Subsystem = "endpoint"
 	SubsysPruning      Subsystem = "pruning"

daemon/snapper/cron.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 	"github.com/zrepl/zrepl/util/suspendresumesafetimer"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -42,7 +43,7 @@ type Cron struct {
 	wakeupWhileRunningCount int
 }
 
-func (s *Cron) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
+func (s *Cron) Run(ctx context.Context, snapshotsTaken *trigger.Manual) {
 
 	for {
 		now := time.Now()
@@ -75,13 +76,7 @@ func (s *Cron) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
 			s.running = false
 			s.mtx.Unlock()
 
-			select {
+			snapshotsTaken.Fire()
-			case snapshotsTaken <- struct{}{}:
-			default:
-				if snapshotsTaken != nil {
-					getLogger(ctx).Warn("callback channel is full, discarding snapshot update event")
-				}
-			}
 		}()
 	}
 

daemon/snapper/manual.go

@@ -2,11 +2,13 @@ package snapper
 
 import (
 	"context"
+
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 )
 
 type manual struct{}
 
-func (s *manual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
+func (s *manual) Run(ctx context.Context, snapshotsTaken *trigger.Manual) {
 	// nothing to do
 }
 

daemon/snapper/periodic.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
@@ -51,7 +52,7 @@ type periodicArgs struct {
 	interval       time.Duration
 	fsf            zfs.DatasetFilter
 	planArgs       planArgs
-	snapshotsTaken chan<- struct{}
+	snapshotsTaken *trigger.Manual
 	dryRun         bool
 }
 
@@ -103,7 +104,7 @@ func (s State) sf() state {
 type updater func(u func(*Periodic)) State
 type state func(a periodicArgs, u updater) state
 
-func (s *Periodic) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
+func (s *Periodic) Run(ctx context.Context, snapshotsTaken *trigger.Manual) {
 	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	getLogger(ctx).Debug("start")
 	defer getLogger(ctx).Debug("stop")
@@ -207,13 +208,7 @@ func periodicStateSnapshot(a periodicArgs, u updater) state {
 
 	ok := plan.execute(a.ctx, false)
 
-	select {
+	a.snapshotsTaken.Fire()
-	case a.snapshotsTaken <- struct{}{}:
-	default:
-		if a.snapshotsTaken != nil {
-			getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
-		}
-	}
 
 	return u(func(snapper *Periodic) {
 		if !ok {

daemon/snapper/snapper.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job/trigger"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -17,7 +18,7 @@ const (
 )
 
 type Snapper interface {
-	Run(ctx context.Context, snapshotsTaken chan<- struct{})
+	Run(ctx context.Context, snapshotsTaken *trigger.Manual)
 	Report() Report
 }
 

docs/configuration/overview.rst

@@ -130,7 +130,7 @@ The following high-level steps take place during replication and can be monitore
   * Move the **replication cursor** bookmark on the sending side (see below).
   * Move the **last-received-hold** on the receiving side (see below).
   * Release the send-side step-holds.
-   
+
 The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated.
 
 ZFS Background Knowledge
@@ -258,6 +258,9 @@ On your setup, ensure that
 
 * all ``filesystems`` filter specifications are disjoint
 * no ``root_fs`` is a prefix or equal to another ``root_fs``
+
+  * For ``sink`` jobs, consider all possible ``root_fs/${client_identity}``.
+
 * no ``filesystems`` filter matches any ``root_fs``
 
 **Exceptions to the rule**:

packaging/rpm/zrepl.spec

@@ -16,7 +16,7 @@ zrepl is a one-stop, integrated solution for ZFS replication.}
 %define __strip /usr/bin/true
 
 Name:             zrepl
-Release:          1
+Release:          SUBSTITUTED_BY_MAKEFILE
 Summary:          One-stop, integrated solution for ZFS replication
 License:          MIT
 URL:              https://zrepl.github.io/

pruning/keep_last_n.go

@@ -33,11 +33,6 @@ func NewKeepLastN(n int, regex string) (*KeepLastN, error) {
 }
 
 func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
-
-	if k.n > len(snaps) {
-		return []Snapshot{}
-	}
-
 	matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
 		return k.re.MatchString(snapshot.Name())
 	})

pruning/keep_last_n_test.go

@@ -90,7 +90,7 @@ func TestKeepLastN(t *testing.T) {
 				stubSnap{"a2", false, o(12)},
 			},
 			rules: []KeepRule{
-				MustKeepLastN(3, "a"),
+				MustKeepLastN(4, "a"),
 			},
 			expDestroy: map[string]bool{
 				"b1": true,