replication/driver: simplify second-attempt step correlation code & fix statekeeping

Before this change, the step correlation code returned early in several cases:
- did not set f.planning.done in the cases where it was a no-op
- did not set f.planning.err in the cases where correlation did not
  succeed

Reported-by: InsanePrawn <insane.prawny@gmail.com>

.circleci/config.yml

@@ -204,7 +204,7 @@ workflows:
   periodic:
     triggers:
       - schedule:
-          cron: "45 15 * * *"
+          cron: "00 17 * * *"
           filters:
             branches:
               only:
@@ -212,7 +212,7 @@ workflows:
                 - stable
                 - problame/circleci-build
     jobs:
-      - periodic-trigger-pipeline
+      - periodic-full-pipeline-run
 
   zrepl.github.io:
     jobs:
@@ -342,14 +342,12 @@ jobs:
           description: CI-generated release artifacts
           minio-dst: ""
 
-  periodic-trigger-pipeline:
+  periodic-full-pipeline-run:
     docker:
       - image: cimg/base:2020.08
     steps:
       - trigger-pipeline:
-          body_no_shell_subst: '{"branch":"master", "parameters": { "do_ci": true, "do_release": true }}'
+          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
-      - trigger-pipeline:
-          body_no_shell_subst: '{"branch":"stable", "parameters": { "do_ci": true, "do_release": true }}'
 
   publish-zrepl-github-io:
     docker:

README.md

@@ -1,7 +1,7 @@
 [![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.vercel.app%2Fapi%3Fusername%3Dzrepl%26type%3Dpatrons&style=flat&color=yellow)](https://patreon.com/zrepl)
+[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
 [![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
 [![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
 [![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)

docs/configuration/overview.rst

@@ -188,7 +188,7 @@ Encoding the job name in the names ensures that multiple sending jobs can replic
 
 .. _tentative-replication-cursor-bookmarks:
 
-**Tentative replication cursor bookmarks** are short-lived boomkarks that protect the atomic moving-forward of the replication cursor and last-received-hold (see :issue:`this issue <340>`).
+**Tentative replication cursor bookmarks** are short-lived bookmarks that protect the atomic moving-forward of the replication cursor and last-received-hold (see :issue:`this issue <340>`).
 They are only necessary if step holds are not used as per the :ref:`replication.protection <replication-option-protection>` setting.
 The tentative replication cursor has the format ``#zrepl_CUSORTENTATIVE_G_<GUID>_J_<JOBNAME>``.
 The ``zrepl zfs-abstraction list`` command provides a listing of all bookmarks and holds managed by zrepl.
@@ -255,7 +255,7 @@ On your setup, ensure that
 
 * all ``filesystems`` filter specifications are disjoint
 * no ``root_fs`` is a prefix or equal to another ``root_fs``
-* no ``filesystems`` filter maches any ``root_fs``
+* no ``filesystems`` filter matches any ``root_fs``
 
 **Exceptions to the rule**:
 

docs/global.rst.inc

@@ -13,7 +13,7 @@
    :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
 .. |Donate via Liberapay| image:: https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay
    :target: https://liberapay.com/zrepl/donate
-.. |Donate via Patreon| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.vercel.app%2Fapi%3Fusername%3Dzrepl%26type%3Dpatrons&style=flat&color=yellow
+.. |Donate via Patreon| image:: https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079
    :target: https://www.patreon.com/zrepl
 .. |Donate via GitHub Sponsors| image:: https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow
    :target: https://github.com/sponsors/problame

docs/supporters.rst

@@ -32,6 +32,7 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
+* |supporter-gold| Cyberiada GmbH
 * |supporter-std| `Gordon Schulz <https://github.com/azmodude>`_
 * |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
 * |supporter-std| `Michael D. Schmitt <https://waterbendingscroll.dancingdragons.org>`_

replication/driver/replication_driver.go

@@ -492,57 +492,37 @@ func (f *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
 	// => don't set f.planning.done just yet
 	f.debug("initial len(fs.planned.steps) = %d", len(f.planned.steps))
 
-	// for not-first attempts, only allow fs.planned.steps
+	// for not-first attempts that succeeded in planning, only allow fs.planned.steps
-	// up to including the originally planned target snapshot
+	// up to and including the originally planned target snapshot
 	if prev != nil && prev.planning.done && prev.planning.err == nil {
+		f.debug("attempting to correlate plan with previous attempt to find out what is left to do")
+		// find the highest of the previously uncompleted steps for which we can also find a step
+		// in our current plan
 		prevUncompleted := prev.planned.steps[prev.planned.step:]
-		if len(prevUncompleted) == 0 {
+		var target struct{ prev, cur int }
-			f.debug("prevUncompleted is empty")
+		target.prev = -1
-			return
+		target.cur = -1
-		}
+	out:
-		if len(f.planned.steps) == 0 {
+		for p := len(prevUncompleted) - 1; p >= 0; p-- {
-			f.debug("fs.planned.steps is empty")
+			for q := len(f.planned.steps) - 1; q >= 0; q-- {
-			return
+				if prevUncompleted[p].step.TargetEquals(f.planned.steps[q].step) {
-		}
+					target.prev = p
-		prevFailed := prevUncompleted[0]
+					target.cur = q
-		curFirst := f.planned.steps[0]
+					break out
-		// we assume that PlanFS retries prevFailed (using curFirst)
-		if !prevFailed.step.TargetEquals(curFirst.step) {
-			f.debug("Targets don't match")
-			// Two options:
-			// A: planning algorithm is broken
-			// B: manual user intervention inbetween
-			// Neither way will we make progress, so let's error out
-			stepFmt := func(step *step) string {
-				r := step.report()
-				s := r.Info
-				if r.IsIncremental() {
-					return fmt.Sprintf("%s=>%s", s.From, s.To)
-				} else {
-					return fmt.Sprintf("full=>%s", s.To)
 				}
 			}
-			msg := fmt.Sprintf("last attempt's uncompleted step %s does not correspond to this attempt's first planned step %s",
+		}
-				stepFmt(prevFailed), stepFmt(curFirst))
+		if target.prev == -1 || target.cur == -1 {
-			f.planned.stepErr = newTimedError(errors.New(msg), time.Now())
+			f.debug("no correlation possible between previous attempt and this attempt's plan")
+			f.planning.err = newTimedError(fmt.Errorf("cannot correlate previously failed attempt to current plan"), time.Now())
 			return
 		}
-		// only allow until step targets diverge
+
-		min := len(prevUncompleted)
+		f.planned.steps = f.planned.steps[0:target.cur]
-		if min > len(f.planned.steps) {
+		f.debug("found correlation, new steps are len(fs.planned.steps) = %d", len(f.planned.steps))
-			min = len(f.planned.steps)
+	} else {
-		}
+		f.debug("previous attempt does not exist or did not finish planning, no correlation possible, taking this attempt's plan as is")
-		diverge := 0
-		for ; diverge < min; diverge++ {
-			f.debug("diverge compare iteration %d", diverge)
-			if !f.planned.steps[diverge].step.TargetEquals(prevUncompleted[diverge].step) {
-				break
-			}
-		}
-		f.debug("diverge is %d", diverge)
-		f.planned.steps = f.planned.steps[0:diverge]
 	}
-	f.debug("post-prev-merge len(fs.planned.steps) = %d", len(f.planned.steps))
 
 	// now we are done planning (f.planned.steps won't change from now on)
 	f.planning.done = true