config: support for including files & directories (conf.d) (#856)

This PR allows for the distribution of zrepl job definitions across
multiple YAML files that are included from the main config.

```
global: ...
include:
- ./zrepl.yml.d
- /opt/zrepl.yml
- ...
```

Refer to the docs changes for details.

Co-authored-by: Christian Schwarz <me@cschwarz.com>

docs/configuration/overview.rst

@@ -30,6 +30,45 @@ Config File Structure
 A zrepl configuration file is divided in to two main sections: ``global`` and ``jobs``.
 ``global`` has sensible defaults. It is covered in :ref:`logging <logging>`, :ref:`monitoring <monitoring>` \& :ref:`miscellaneous <miscellaneous>`.
 
+
+``conf.d``: including config files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to distribute zrepl configurations over multiple YAML configuration
+files. This is achieved by using the `include` key which can only exist in the main
+configuration file.
+
+The list of included paths must point to either individual YAML files or directories.
+If the path points to a directory then all YAML files with a `.yml` extention in the
+directory will be included.
+
+.. code-block:: yaml
+
+    # /etc/zrepl/zrep.yml
+    global: ...
+    include:
+      - ./jobs.d/
+      - /opt/some_job.yml
+
+    # /etc/zrepl/jobs.d/backup.yml
+    jobs:
+    - name: backup
+      type: push
+    - ...
+
+    # /opt/some_job.yml
+    jobs:
+    - name: another_job
+      ...
+    - ...
+
+
+
+The paths are treated as absolute when starting with `/`, otherwise
+as a relative path to the main config file's parent directory.
+
+Job names must be unique across all included configuration files.
+
 .. _job-overview:
 
 Jobs \& How They Work Together

internal/config/config.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"log/syslog"
 	"os"
+	pathpkg "path"
+	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
@@ -24,6 +26,7 @@ const (
 type Config struct {
 	Jobs    []JobEnum `yaml:"jobs,optional"`
 	Global  *Global   `yaml:"global,optional,fromdefaults"`
+	Include []string  `yaml:"include,optional"`
 }
 
 func (c *Config) Job(name string) (*JobEnum, error) {
@@ -655,8 +658,9 @@ var ConfigFileDefaultLocations = []string{
 	"/usr/local/etc/zrepl/zrepl.yml",
 }
 
-func ParseConfig(path string) (i *Config, err error) {
+func ParseConfig(path string) (rootConfig *Config, err error) {
 
+	// Parse main configuration file
 	if path == "" {
 		// Try default locations
 		for _, l := range ConfigFileDefaultLocations {
@@ -679,7 +683,67 @@ func ParseConfig(path string) (i *Config, err error) {
 		return
 	}
 
-	return ParseConfigBytes(bytes)
+	rootConfig, err = ParseConfigBytes(bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	err = expandConfigInclude(path, rootConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return rootConfig, err
+}
+
+func expandConfigInclude(configPath string, config *Config) (err error) {
+	var includeConfigPaths []string
+	for _, path := range config.Include {
+		if !pathpkg.IsAbs(configPath) {
+			path = pathpkg.Join(pathpkg.Dir(configPath), path)
+		}
+
+		stat, statErr := os.Stat(path)
+		if statErr != nil {
+			return errors.Wrapf(statErr, "stat path %q", path)
+		}
+
+		if stat.Mode().IsDir() {
+			directoryPaths, err := filepath.Glob(path + "/*.yml")
+			if err != nil {
+				return err
+			}
+
+			includeConfigPaths = append(includeConfigPaths, directoryPaths...)
+		} else if stat.Mode().IsRegular() {
+			if extention := filepath.Ext(path); extention != ".yml" {
+				return fmt.Errorf("include config files must end with `.yml`: %s", path)
+			}
+			includeConfigPaths = append(includeConfigPaths, path)
+		} else {
+			return fmt.Errorf("not a file or directory: %s", path)
+		}
+	}
+
+	for _, path := range includeConfigPaths {
+		var bytes []byte
+		if bytes, err = os.ReadFile(path); err != nil {
+			return errors.Wrapf(err, "read file: %q", path)
+		}
+
+		includedConfig, err := ParseConfigBytes(bytes)
+		if err != nil {
+			return err
+		}
+
+		if len(includedConfig.Include) > 0 {
+			return errors.Errorf("included configuration files must not include other files: %s", path)
+		}
+
+		config.Jobs = append(config.Jobs, includedConfig.Jobs...)
+	}
+
+	return nil
 }
 
 func ParseConfigBytes(bytes []byte) (*Config, error) {

internal/config/config_include_test.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIncludeSingle(t *testing.T) {
+	path := "./samples/include.yml"
+
+	t.Run(path, func(t *testing.T) {
+		config, err := ParseConfig(path)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", path, err)
+		}
+
+		require.NotNil(t, config)
+		require.NotNil(t, config.Global)
+		require.NotEmpty(t, config.Jobs)
+
+		t.Logf("file: %s", path)
+		t.Log(pretty.Sprint(config))
+	})
+}
+
+func TestIncludeDirectory(t *testing.T) {
+	path := "./samples/include_directory.yml"
+
+	t.Run(path, func(t *testing.T) {
+		config, err := ParseConfig(path)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", path, err)
+		}
+
+		require.NotNil(t, config)
+		require.NotNil(t, config.Global)
+		require.NotEmpty(t, config.Jobs)
+
+		t.Logf("file: %s", path)
+		t.Log(pretty.Sprint(config))
+	})
+}

internal/config/samples/include.d/snap.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

internal/config/samples/include.yml

@@ -0,0 +1,2 @@
+include:
+  - ./snap.yml

internal/config/samples/include_directory.yml

@@ -0,0 +1,2 @@
+include:
+  - ./include.d