pruner: improve cancellation + error handling strategy

Pruner now backs off as soon as there is an error, making that error the
Error field in the pruner report.
The error is also stored in the specific *fs that failed, and we
maintain an error counter per *fs to de-prioritize those fs that failed.
Like with replication, the de-prioritization on errors is to avoid '
getting stuck' with an individual filesystem until the watchdog hits.

daemon/pruner/pruner_test.go

@@ -133,16 +133,14 @@ func TestPruner_Prune(t *testing.T) {
 		},
 		listVersionsErrs: map[string][]error{
 			"zroot/foo": {
-				stubNetErr{msg: "fakeerror1", temporary: true}, // should be classified as temporaty
+				stubNetErr{msg: "fakeerror1", temporary: true},
 				stubNetErr{msg: "fakeerror2", temporary: true,},
 			},
 		},
 		destroyErrs: map[string][]error{
-			"zroot/foo": {
-				fmt.Errorf("permanent error"),
-			},
-			"zroot/bar": {
-				stubNetErr{msg: "fakeerror3", temporary: true},
+			"zroot/baz": {
+				stubNetErr{msg: "fakeerror3", temporary: true}, // first error puts it back in the queue
+				stubNetErr{msg:"permanent error"}, // so it will be last when pruner gives up due to permanent err
 			},
 		},
 		destroyed: make(map[string][]string),
@@ -178,9 +176,6 @@ func TestPruner_Prune(t *testing.T) {
 			"zroot/foo": {
 				stubNetErr{msg: "fakeerror4", temporary: true},
 			},
-			"zroot/baz": {
-				fmt.Errorf("permanent error2"),
-			},
 		},
 	}
 
@@ -199,9 +194,8 @@ func TestPruner_Prune(t *testing.T) {
 	p.Prune()
 
 	exp := map[string][]string{
+		"zroot/foo": {"drop_c"},
 		"zroot/bar": {"drop_g"},
-		// drop_c is prohibited by failing destroy
-		// drop_i is prohibiteed by failing ReplicationCursor call
 	}
 
 	assert.Equal(t, exp, target.destroyed)