adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into InsanePrawn-master

Browse Source
This commit is contained in:
Christian Schwarz
2019-03-15 21:05:20 +01:00
parent 5c5e8c0baf 5dfe24eeee
commit aff639e87a
114 changed files with 6898 additions and 1675 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/config.go
+51 -20
View File
@@ -5,6 +5,7 @@ import (
"github.com/pkg/errors"
"github.com/zrepl/yaml-config"
"io/ioutil"
"log/syslog"
"os"
"reflect"
"regexp"
@@ -39,7 +40,7 @@ func (j JobEnum) Name() string {
case *PullJob: name = v.Name
case *SourceJob: name = v.Name
default:
panic(fmt.Sprintf("unknownn job type %T", v))
panic(fmt.Sprintf("unknown job type %T", v))
}
return name
}
@@ -140,7 +141,6 @@ type Global struct {
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
Control *GlobalControl `yaml:"control,optional,fromdefaults"`
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
RPC *RPCConfig `yaml:"rpc,optional,fromdefaults"`
}
func Default(i interface{}) {
@@ -155,29 +155,18 @@ func Default(i interface{}) {
}
}
type RPCConfig struct {
Timeout time.Duration `yaml:"timeout,optional,positive,default=10s"`
TxChunkSize uint32 `yaml:"tx_chunk_size,optional,default=32768"`
RxStructuredMaxLen uint32 `yaml:"rx_structured_max,optional,default=16777216"`
RxStreamChunkMaxLen uint32 `yaml:"rx_stream_chunk_max,optional,default=16777216"`
RxHeaderMaxLen uint32 `yaml:"rx_header_max,optional,default=40960"`
SendHeartbeatInterval time.Duration `yaml:"send_heartbeat_interval,optional,positive,default=5s"`
}
type ConnectEnum struct {
Ret interface{}
}
type ConnectCommon struct {
Type string `yaml:"type"`
RPC *RPCConfig `yaml:"rpc,optional"`
Type string `yaml:"type"`
}
type TCPConnect struct {
ConnectCommon `yaml:",inline"`
Address string `yaml:"address"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type TLSConnect struct {
@@ -187,7 +176,7 @@ type TLSConnect struct {
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ServerCN string `yaml:"server_cn"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type SSHStdinserverConnect struct {
@@ -199,7 +188,7 @@ type SSHStdinserverConnect struct {
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
Options []string `yaml:"options,optional"`
DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"`
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
}
type LocalConnect struct {
@@ -213,8 +202,7 @@ type ServeEnum struct {
}
type ServeCommon struct {
Type string `yaml:"type"`
RPC *RPCConfig `yaml:"rpc,optional"`
Type string `yaml:"type"`
}
type TCPServe struct {
@@ -230,7 +218,7 @@ type TLSServe struct {
Cert string `yaml:"cert"`
Key string `yaml:"key"`
ClientCNs []string `yaml:"client_cns"`
HandshakeTimeout time.Duration `yaml:"handshake_timeout,positive,default=10s"`
HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
}
type StdinserverServer struct {
@@ -281,6 +269,7 @@ type StdoutLoggingOutlet struct {
type SyslogLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"`
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
}
@@ -307,6 +296,14 @@ type PrometheusMonitoring struct {
Listen string `yaml:"listen"`
}
type SyslogFacility syslog.Priority
func (f *SyslogFacility) SetDefault() {
*f = SyslogFacility(syslog.LOG_LOCAL0)
}
var _ yaml.Defaulter = (*SyslogFacility)(nil)
type GlobalControl struct {
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
}
@@ -413,6 +410,40 @@ func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err err
return
}
func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
var s string
if err := u(&s, true); err != nil {
return err
}
var level syslog.Priority
switch s {
case "kern": level = syslog.LOG_KERN
case "user": level = syslog.LOG_USER
case "mail": level = syslog.LOG_MAIL
case "daemon": level = syslog.LOG_DAEMON
case "auth": level = syslog.LOG_AUTH
case "syslog": level = syslog.LOG_SYSLOG
case "lpr": level = syslog.LOG_LPR
case "news": level = syslog.LOG_NEWS
case "uucp": level = syslog.LOG_UUCP
case "cron": level = syslog.LOG_CRON
case "authpriv": level = syslog.LOG_AUTHPRIV
case "ftp": level = syslog.LOG_FTP
case "local0": level = syslog.LOG_LOCAL0
case "local1": level = syslog.LOG_LOCAL1
case "local2": level = syslog.LOG_LOCAL2
case "local3": level = syslog.LOG_LOCAL3
case "local4": level = syslog.LOG_LOCAL4
case "local5": level = syslog.LOG_LOCAL5
case "local6": level = syslog.LOG_LOCAL6
case "local7": level = syslog.LOG_LOCAL7
default:
return fmt.Errorf("invalid syslog level: %q", s)
}
*t = SyslogFacility(level)
return nil
}
var ConfigFileDefaultLocations = []string{
"/etc/zrepl/zrepl.yml",
"/usr/local/etc/zrepl/zrepl.yml",
config/config_global_test.go
+32
View File
@@ -1,9 +1,11 @@
package config
import (
"fmt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zrepl/yaml-config"
"log/syslog"
"testing"
)
@@ -72,6 +74,36 @@ global:
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
}
func TestSyslogLoggingOutletFacility(t *testing.T) {
type SyslogFacilityPriority struct {
Facility string
Priority syslog.Priority
}
syslogFacilitiesPriorities := []SyslogFacilityPriority{
{"", syslog.LOG_LOCAL0}, // default
{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
}
for _, sFP := range syslogFacilitiesPriorities {
logcfg := fmt.Sprintf(`
global:
logging:
- type: syslog
level: info
format: human
facility: %s
`, sFP.Facility)
conf := testValidGlobalSection(t, logcfg)
assert.Equal(t, 1, len(*conf.Global.Logging))
assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
}
}
func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
e := &LoggingOutletEnumList{}
var i yaml.Defaulter = e
config/config_rpc_test.go
-86
View File
@@ -1,86 +0,0 @@
package config
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestRPC(t *testing.T) {
conf := testValidConfig(t, `
jobs:
- name: pull_servers
type: pull
connect:
type: tcp
address: "server1.foo.bar:8888"
rpc:
timeout: 20s # different form default, should merge
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: last_n
count: 100
- name: pull_servers2
type: pull
connect:
type: tcp
address: "server1.foo.bar:8888"
rpc:
tx_chunk_size: 0xabcd # different from default, should merge
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: last_n
count: 100
- type: sink
name: "laptop_sink"
root_fs: "pool2/backup_laptops"
serve:
type: tcp
listen: "192.168.122.189:8888"
clients: {
"10.23.42.23":"client1"
}
rpc:
rx_structured_max: 0x2342
- type: sink
name: "other_sink"
root_fs: "pool2/backup_laptops"
serve:
type: tcp
listen: "192.168.122.189:8888"
clients: {
"10.23.42.23":"client1"
}
rpc:
send_heartbeat_interval: 10s
`)
assert.Equal(t, 20*time.Second, conf.Jobs[0].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.Timeout)
assert.Equal(t, uint32(0xabcd), conf.Jobs[1].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.TxChunkSize)
assert.Equal(t, uint32(0x2342), conf.Jobs[2].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.RxStructuredMaxLen)
assert.Equal(t, 10*time.Second, conf.Jobs[3].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.SendHeartbeatInterval)
defConf := RPCConfig{}
Default(&defConf)
assert.Equal(t, defConf.Timeout, conf.Global.RPC.Timeout)
}
func TestGlobal_DefaultRPCConfig(t *testing.T) {
assert.NotPanics(t, func() {
var c RPCConfig
Default(&c)
assert.NotNil(t, c)
assert.Equal(t, c.TxChunkSize, uint32(1)<<15)
})
}
config/config_test.go
+20 -4
View File
@@ -1,11 +1,14 @@
package config
import (
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
"bytes"
"path"
"path/filepath"
"testing"
"text/template"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
@@ -35,8 +38,21 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
}
// template must be a template/text template with a single '{{ . }}' as placehodler for val
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
tmp, err := template.New("master").Parse(tmpl)
if err != nil {
panic(err)
}
var buf bytes.Buffer
err = tmp.Execute(&buf, val)
if err != nil {
panic(err)
}
return testValidConfig(t, buf.String())
}
func testValidConfig(t *testing.T, input string) (*Config) {
func testValidConfig(t *testing.T, input string) *Config {
t.Helper()
conf, err := testConfig(t, input)
require.NoError(t, err)
@@ -47,4 +63,4 @@ func testValidConfig(t *testing.T, input string) (*Config) {
func testConfig(t *testing.T, input string) (*Config, error) {
t.Helper()
return ParseConfigBytes([]byte(input))
}
}