From 4d6583ea5f6c7acc6d33c53f1f57c9048843c630 Mon Sep 17 00:00:00 2001 From: Zeyad Tamimi Date: Mon, 19 Jan 2026 00:19:54 -0800 Subject: [PATCH] config: support for including files & directories (`conf.d`) (#856) This PR allows for the distribution of zrepl job definitions across multiple YAML files that are included from the main config. ``` global: ... include: - ./zrepl.yml.d - /opt/zrepl.yml - ... ``` Refer to the docs changes for details. Co-authored-by: Christian Schwarz --- docs/configuration/overview.rst | 39 ++++++++++ internal/config/config.go | 72 +++++++++++++++++-- internal/config/config_include_test.go | 44 ++++++++++++ internal/config/samples/include.d/snap.yml | 14 ++++ internal/config/samples/include.yml | 2 + internal/config/samples/include_directory.yml | 2 + 6 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 internal/config/config_include_test.go create mode 100644 internal/config/samples/include.d/snap.yml create mode 100644 internal/config/samples/include.yml create mode 100644 internal/config/samples/include_directory.yml diff --git a/docs/configuration/overview.rst b/docs/configuration/overview.rst index 4b9de34..c7e9d34 100644 --- a/docs/configuration/overview.rst +++ b/docs/configuration/overview.rst @@ -30,6 +30,45 @@ Config File Structure A zrepl configuration file is divided in to two main sections: ``global`` and ``jobs``. ``global`` has sensible defaults. It is covered in :ref:`logging `, :ref:`monitoring ` \& :ref:`miscellaneous `. + +``conf.d``: including config files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to distribute zrepl configurations over multiple YAML configuration +files. This is achieved by using the `include` key which can only exist in the main +configuration file. + +The list of included paths must point to either individual YAML files or directories. +If the path points to a directory then all YAML files with a `.yml` extention in the +directory will be included. + +.. code-block:: yaml + + # /etc/zrepl/zrep.yml + global: ... + include: + - ./jobs.d/ + - /opt/some_job.yml + + # /etc/zrepl/jobs.d/backup.yml + jobs: + - name: backup + type: push + - ... + + # /opt/some_job.yml + jobs: + - name: another_job + ... + - ... + + + +The paths are treated as absolute when starting with `/`, otherwise +as a relative path to the main config file's parent directory. + +Job names must be unique across all included configuration files. + .. _job-overview: Jobs \& How They Work Together diff --git a/internal/config/config.go b/internal/config/config.go index 7722c6a..583380c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,8 @@ import ( "fmt" "log/syslog" "os" + pathpkg "path" + "path/filepath" "time" "github.com/pkg/errors" @@ -22,8 +24,9 @@ const ( ) type Config struct { - Jobs []JobEnum `yaml:"jobs,optional"` - Global *Global `yaml:"global,optional,fromdefaults"` + Jobs []JobEnum `yaml:"jobs,optional"` + Global *Global `yaml:"global,optional,fromdefaults"` + Include []string `yaml:"include,optional"` } func (c *Config) Job(name string) (*JobEnum, error) { @@ -655,8 +658,9 @@ var ConfigFileDefaultLocations = []string{ "/usr/local/etc/zrepl/zrepl.yml", } -func ParseConfig(path string) (i *Config, err error) { +func ParseConfig(path string) (rootConfig *Config, err error) { + // Parse main configuration file if path == "" { // Try default locations for _, l := range ConfigFileDefaultLocations { @@ -679,7 +683,67 @@ func ParseConfig(path string) (i *Config, err error) { return } - return ParseConfigBytes(bytes) + rootConfig, err = ParseConfigBytes(bytes) + if err != nil { + return nil, err + } + + err = expandConfigInclude(path, rootConfig) + if err != nil { + return nil, err + } + + return rootConfig, err +} + +func expandConfigInclude(configPath string, config *Config) (err error) { + var includeConfigPaths []string + for _, path := range config.Include { + if !pathpkg.IsAbs(configPath) { + path = pathpkg.Join(pathpkg.Dir(configPath), path) + } + + stat, statErr := os.Stat(path) + if statErr != nil { + return errors.Wrapf(statErr, "stat path %q", path) + } + + if stat.Mode().IsDir() { + directoryPaths, err := filepath.Glob(path + "/*.yml") + if err != nil { + return err + } + + includeConfigPaths = append(includeConfigPaths, directoryPaths...) + } else if stat.Mode().IsRegular() { + if extention := filepath.Ext(path); extention != ".yml" { + return fmt.Errorf("include config files must end with `.yml`: %s", path) + } + includeConfigPaths = append(includeConfigPaths, path) + } else { + return fmt.Errorf("not a file or directory: %s", path) + } + } + + for _, path := range includeConfigPaths { + var bytes []byte + if bytes, err = os.ReadFile(path); err != nil { + return errors.Wrapf(err, "read file: %q", path) + } + + includedConfig, err := ParseConfigBytes(bytes) + if err != nil { + return err + } + + if len(includedConfig.Include) > 0 { + return errors.Errorf("included configuration files must not include other files: %s", path) + } + + config.Jobs = append(config.Jobs, includedConfig.Jobs...) + } + + return nil } func ParseConfigBytes(bytes []byte) (*Config, error) { diff --git a/internal/config/config_include_test.go b/internal/config/config_include_test.go new file mode 100644 index 0000000..3a95648 --- /dev/null +++ b/internal/config/config_include_test.go @@ -0,0 +1,44 @@ +package config + +import ( + "testing" + + "github.com/kr/pretty" + "github.com/stretchr/testify/require" +) + +func TestIncludeSingle(t *testing.T) { + path := "./samples/include.yml" + + t.Run(path, func(t *testing.T) { + config, err := ParseConfig(path) + if err != nil { + t.Errorf("error parsing %s:\n%+v", path, err) + } + + require.NotNil(t, config) + require.NotNil(t, config.Global) + require.NotEmpty(t, config.Jobs) + + t.Logf("file: %s", path) + t.Log(pretty.Sprint(config)) + }) +} + +func TestIncludeDirectory(t *testing.T) { + path := "./samples/include_directory.yml" + + t.Run(path, func(t *testing.T) { + config, err := ParseConfig(path) + if err != nil { + t.Errorf("error parsing %s:\n%+v", path, err) + } + + require.NotNil(t, config) + require.NotNil(t, config.Global) + require.NotEmpty(t, config.Jobs) + + t.Logf("file: %s", path) + t.Log(pretty.Sprint(config)) + }) +} diff --git a/internal/config/samples/include.d/snap.yml b/internal/config/samples/include.d/snap.yml new file mode 100644 index 0000000..6775b01 --- /dev/null +++ b/internal/config/samples/include.d/snap.yml @@ -0,0 +1,14 @@ +jobs: +- name: snapjob + type: snap + filesystems: { + "tank/frequently_changed<": true, + } + snapshotting: + type: periodic + interval: 2m + prefix: zrepl_snapjob_ + pruning: + keep: + - type: last_n + count: 60 diff --git a/internal/config/samples/include.yml b/internal/config/samples/include.yml new file mode 100644 index 0000000..c281c52 --- /dev/null +++ b/internal/config/samples/include.yml @@ -0,0 +1,2 @@ +include: + - ./snap.yml diff --git a/internal/config/samples/include_directory.yml b/internal/config/samples/include_directory.yml new file mode 100644 index 0000000..a6795f9 --- /dev/null +++ b/internal/config/samples/include_directory.yml @@ -0,0 +1,2 @@ +include: + - ./include.d