adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rpc rewrite: control RPCs using gRPC + separate RPC for data transfer

Browse Source 
transport/ssh: update go-netssh to new version
    => supports CloseWrite and Deadlines
    => build: require Go 1.11 (netssh requires it)
This commit is contained in:
Christian Schwarz
2018-12-11 22:01:50 +01:00
parent d281fb00e3
commit 796c5ad42d
100 changed files with 6460 additions and 1485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/config.go
+2 -15
View File
@@ -130,7 +130,6 @@ type Global struct {
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
Control *GlobalControl `yaml:"control,optional,fromdefaults"`
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
RPC *RPCConfig `yaml:"rpc,optional,fromdefaults"`
}
func Default(i interface{}) {
@@ -145,23 +144,12 @@ func Default(i interface{}) {
}
}
type RPCConfig struct {
Timeout time.Duration `yaml:"timeout,optional,positive,default=10s"`
TxChunkSize uint32 `yaml:"tx_chunk_size,optional,default=32768"`
RxStructuredMaxLen uint32 `yaml:"rx_structured_max,optional,default=16777216"`
RxStreamChunkMaxLen uint32 `yaml:"rx_stream_chunk_max,optional,default=16777216"`
RxHeaderMaxLen uint32 `yaml:"rx_header_max,optional,default=40960"`
SendHeartbeatInterval time.Duration `yaml:"send_heartbeat_interval,optional,positive,default=5s"`
}
type ConnectEnum struct {
Ret interface{}
}
type ConnectCommon struct {
Type string `yaml:"type"`
RPC *RPCConfig `yaml:"rpc,optional"`
Type string `yaml:"type"`
}
type TCPConnect struct {
@@ -203,8 +191,7 @@ type ServeEnum struct {
}
type ServeCommon struct {
Type string `yaml:"type"`
RPC *RPCConfig `yaml:"rpc,optional"`
Type string `yaml:"type"`
}
type TCPServe struct {
config/config_rpc_test.go
-86
View File
@@ -1,86 +0,0 @@
package config
import (
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestRPC(t *testing.T) {
conf := testValidConfig(t, `
jobs:
- name: pull_servers
type: pull
connect:
type: tcp
address: "server1.foo.bar:8888"
rpc:
timeout: 20s # different form default, should merge
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: last_n
count: 100
- name: pull_servers2
type: pull
connect:
type: tcp
address: "server1.foo.bar:8888"
rpc:
tx_chunk_size: 0xabcd # different from default, should merge
root_fs: "pool2/backup_servers"
interval: 10m
pruning:
keep_sender:
- type: not_replicated
keep_receiver:
- type: last_n
count: 100
- type: sink
name: "laptop_sink"
root_fs: "pool2/backup_laptops"
serve:
type: tcp
listen: "192.168.122.189:8888"
clients: {
"10.23.42.23":"client1"
}
rpc:
rx_structured_max: 0x2342
- type: sink
name: "other_sink"
root_fs: "pool2/backup_laptops"
serve:
type: tcp
listen: "192.168.122.189:8888"
clients: {
"10.23.42.23":"client1"
}
rpc:
send_heartbeat_interval: 10s
`)
assert.Equal(t, 20*time.Second, conf.Jobs[0].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.Timeout)
assert.Equal(t, uint32(0xabcd), conf.Jobs[1].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.TxChunkSize)
assert.Equal(t, uint32(0x2342), conf.Jobs[2].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.RxStructuredMaxLen)
assert.Equal(t, 10*time.Second, conf.Jobs[3].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.SendHeartbeatInterval)
defConf := RPCConfig{}
Default(&defConf)
assert.Equal(t, defConf.Timeout, conf.Global.RPC.Timeout)
}
func TestGlobal_DefaultRPCConfig(t *testing.T) {
assert.NotPanics(t, func() {
var c RPCConfig
Default(&c)
assert.NotNil(t, c)
assert.Equal(t, c.TxChunkSize, uint32(1)<<15)
})
}
config/config_test.go
+20 -4
View File
@@ -1,11 +1,14 @@
package config
import (
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
"bytes"
"path"
"path/filepath"
"testing"
"text/template"
"github.com/kr/pretty"
"github.com/stretchr/testify/require"
)
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
@@ -35,8 +38,21 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
}
// template must be a template/text template with a single '{{ . }}' as placehodler for val
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
tmp, err := template.New("master").Parse(tmpl)
if err != nil {
panic(err)
}
var buf bytes.Buffer
err = tmp.Execute(&buf, val)
if err != nil {
panic(err)
}
return testValidConfig(t, buf.String())
}
func testValidConfig(t *testing.T, input string) (*Config) {
func testValidConfig(t *testing.T, input string) *Config {
t.Helper()
conf, err := testConfig(t, input)
require.NoError(t, err)
@@ -47,4 +63,4 @@ func testValidConfig(t *testing.T, input string) (*Config) {
func testConfig(t *testing.T, input string) (*Config, error) {
t.Helper()
return ParseConfigBytes([]byte(input))
}
}