This commit is contained in:
Christian Schwarz
2018-08-25 21:30:25 +02:00
parent e082816de5
commit e30ae972f4
27 changed files with 130 additions and 147 deletions
+18 -20
View File
@@ -9,8 +9,8 @@ import (
"time"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/util"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/util"
)
type logNetConnConnecter struct {
@@ -29,36 +29,35 @@ func (l logNetConnConnecter) Connect(ctx context.Context) (net.Conn, error) {
}
type logListenerFactory struct {
ListenerFactory
ReadDump, WriteDump string
ListenerFactory
ReadDump, WriteDump string
}
var _ ListenerFactory = logListenerFactory{}
type logListener struct {
net.Listener
ReadDump, WriteDump string
net.Listener
ReadDump, WriteDump string
}
var _ net.Listener = logListener{}
func (m logListenerFactory) Listen() (net.Listener, error) {
l, err := m.ListenerFactory.Listen()
if err != nil {
return nil, err
}
return logListener{l, m.ReadDump, m.WriteDump}, nil
l, err := m.ListenerFactory.Listen()
if err != nil {
return nil, err
}
return logListener{l, m.ReadDump, m.WriteDump}, nil
}
func (l logListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return util.NewNetConnLogger(conn, l.ReadDump, l.WriteDump)
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return util.NewNetConnLogger(conn, l.ReadDump, l.WriteDump)
}
type netsshAddr struct{}
func (netsshAddr) Network() string { return "netssh" }
@@ -78,7 +77,7 @@ func (netsshConnToNetConnAdatper) SetReadDeadline(t time.Time) error { return ni
func (netsshConnToNetConnAdatper) SetWriteDeadline(t time.Time) error { return nil }
type streamrpcLogAdaptor = twoClassLogAdaptor
type streamrpcLogAdaptor = twoClassLogAdaptor
type replicationLogAdaptor = twoClassLogAdaptor
type twoClassLogAdaptor struct {
@@ -87,7 +86,7 @@ type twoClassLogAdaptor struct {
var _ streamrpc.Logger = twoClassLogAdaptor{}
func (a twoClassLogAdaptor) Errorf(fmtStr string, args... interface{}) {
func (a twoClassLogAdaptor) Errorf(fmtStr string, args ...interface{}) {
const errorSuffix = ": %s"
if len(args) == 1 {
if err, ok := args[0].(error); ok && strings.HasSuffix(fmtStr, errorSuffix) {
@@ -99,7 +98,6 @@ func (a twoClassLogAdaptor) Errorf(fmtStr string, args... interface{}) {
a.Logger.Error(fmt.Sprintf(fmtStr, args...))
}
func (a twoClassLogAdaptor) Infof(fmtStr string, args... interface{}) {
func (a twoClassLogAdaptor) Infof(fmtStr string, args ...interface{}) {
a.Logger.Info(fmt.Sprintf(fmtStr, args...))
}
+4 -4
View File
@@ -11,8 +11,8 @@ import (
"github.com/pkg/errors"
"github.com/problame/go-netssh"
"github.com/problame/go-streamrpc"
"time"
"github.com/zrepl/zrepl/cmd/tlsconf"
"time"
)
type SSHStdinserverConnecter struct {
@@ -51,12 +51,12 @@ func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverCo
}
type netsshConnToConn struct { *netssh.SSHConn }
type netsshConnToConn struct{ *netssh.SSHConn }
var _ net.Conn = netsshConnToConn{}
func (netsshConnToConn) SetDeadline(dl time.Time) error { return nil }
func (netsshConnToConn) SetReadDeadline(dl time.Time) error { return nil }
func (netsshConnToConn) SetDeadline(dl time.Time) error { return nil }
func (netsshConnToConn) SetReadDeadline(dl time.Time) error { return nil }
func (netsshConnToConn) SetWriteDeadline(dl time.Time) error { return nil }
func (c *SSHStdinserverConnecter) Connect(dialCtx context.Context) (net.Conn, error) {
+1 -1
View File
@@ -5,10 +5,10 @@ import (
"context"
"encoding/json"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/logger"
"io"
"net"
"net/http"
"github.com/zrepl/zrepl/logger"
)
type ControlJob struct {
+2 -2
View File
@@ -6,10 +6,10 @@ import (
"context"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/zfs"
"sync"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/cmd/endpoint"
)
type LocalJob struct {
+3 -4
View File
@@ -11,8 +11,8 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/replication"
)
type PullJob struct {
@@ -28,7 +28,7 @@ type PullJob struct {
Debug JobDebugSettings
task *Task
rep *replication.Replication
rep *replication.Replication
}
func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
@@ -155,7 +155,6 @@ var STREAMRPC_CONFIG = &streamrpc.ConnConfig{ // FIXME oversight and configurabi
},
}
func (j *PullJob) doRun(ctx context.Context) {
j.task.Enter("run")
@@ -184,7 +183,7 @@ func (j *PullJob) doRun(ctx context.Context) {
}
ctx = replication.WithLogger(ctx, replicationLogAdaptor{j.task.Log().WithField("subsystem", "replication")})
ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{j.task.Log().WithField("subsystem", "rpc.protocol")})
ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{j.task.Log().WithField("subsystem", "rpc.protocol")})
ctx = endpoint.WithLogger(ctx, j.task.Log().WithField("subsystem", "rpc.endpoint"))
j.rep = replication.NewReplication()
+2 -2
View File
@@ -7,8 +7,8 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"net"
"github.com/zrepl/zrepl/cmd/endpoint"
"net"
)
type SourceJob struct {
@@ -160,7 +160,7 @@ func (j *SourceJob) serve(ctx context.Context, task *Task) {
connChan := make(chan connChanMsg)
// Serve connections until interrupted or error
outer:
outer:
for {
go func() {
+3 -3
View File
@@ -1,15 +1,15 @@
package cmd
import (
"crypto/tls"
"crypto/x509"
"github.com/mattn/go-isatty"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/tlsconf"
"github.com/zrepl/zrepl/logger"
"os"
"time"
"crypto/tls"
"crypto/x509"
"github.com/zrepl/zrepl/cmd/tlsconf"
)
type LoggingConfig struct {
+3 -3
View File
@@ -6,8 +6,8 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/zrepl/zrepl/zfs"
"github.com/zrepl/zrepl/cmd/endpoint"
"github.com/zrepl/zrepl/zfs"
)
type DatasetMapFilter struct {
@@ -210,8 +210,8 @@ func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
}
inv.entries[0] = datasetMapFilterEntry{
path: mp,
mapping: e.path.ToString(),
path: mp,
mapping: e.path.ToString(),
subtreeMatch: e.subtreeMatch,
}
+2 -2
View File
@@ -7,12 +7,12 @@ import (
yaml "github.com/go-yaml/yaml"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/replication"
"os"
"regexp"
"strconv"
"time"
"github.com/zrepl/zrepl/replication"
"github.com/problame/go-streamrpc"
)
var ConfigFileDefaultLocations []string = []string{
+3 -3
View File
@@ -22,8 +22,8 @@ type TCPListenerFactory struct {
func parseTCPListenerFactory(c JobParsingContext, i map[string]interface{}) (*TCPListenerFactory, error) {
var in struct {
Address string
TLS map[string]interface{}
Address string
TLS map[string]interface{}
}
if err := mapstructure.Decode(i, &in); err != nil {
return nil, errors.Wrap(err, "mapstructure error")
@@ -56,7 +56,7 @@ func parseTCPListenerFactory(c JobParsingContext, i map[string]interface{}) (*TC
lf.clientCA, err = tlsconf.ParseCAFile(in.CA)
if err != nil {
return errors.Wrap(err,"cannot parse ca file")
return errors.Wrap(err, "cannot parse ca file")
}
lf.serverCert, err = tls.LoadX509KeyPair(in.Cert, in.Key)
+2 -2
View File
@@ -136,7 +136,7 @@ func TestDatasetMapFilter(t *testing.T) {
expectMapping(map1, "q/r", "root4/1/2/r")
map2 := map[string]string{ // identity mapping
"<":"",
"<": "",
}
expectMapping(map2, "foo/bar", "foo/bar")
@@ -280,4 +280,4 @@ func TestDatasetMapFilter_Invert(t *testing.T) {
expectMapping(inv, "foo/bar", "", false, true)
expectMapping(inv, "foo/bar/bee", "bee", false, false)
}
}
+18 -19
View File
@@ -2,20 +2,20 @@
package endpoint
import (
"github.com/zrepl/zrepl/replication/pdu"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/zfs"
"io"
"github.com/pkg/errors"
"github.com/golang/protobuf/proto"
"bytes"
"context"
"github.com/golang/protobuf/proto"
"github.com/pkg/errors"
"github.com/problame/go-streamrpc"
"github.com/zrepl/zrepl/replication"
"github.com/zrepl/zrepl/replication/pdu"
"github.com/zrepl/zrepl/zfs"
"io"
)
// Sender implements replication.ReplicationEndpoint for a sending side
type Sender struct {
FSFilter zfs.DatasetFilter
FSFilter zfs.DatasetFilter
FilesystemVersionFilter zfs.FilesystemVersionFilter
}
@@ -87,16 +87,16 @@ type FSFilter interface {
// FIXME: can we get away without error types here?
type FSMap interface {
FSFilter
Map(path *zfs.DatasetPath) (*zfs.DatasetPath,error)
Invert() (FSMap,error)
AsFilter() (FSFilter)
Map(path *zfs.DatasetPath) (*zfs.DatasetPath, error)
Invert() (FSMap, error)
AsFilter() FSFilter
}
// Receiver implements replication.ReplicationEndpoint for a receiving side
type Receiver struct {
fsmapInv FSMap
fsmap FSMap
fsvf zfs.FilesystemVersionFilter
fsmap FSMap
fsvf zfs.FilesystemVersionFilter
}
func NewReceiver(fsmap FSMap, fsvf zfs.FilesystemVersionFilter) (*Receiver, error) {
@@ -219,12 +219,11 @@ func (e *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, sendStream
// RPC STUBS
// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
const (
RPCListFilesystems = "ListFilesystems"
RPCListFilesystems = "ListFilesystems"
RPCListFilesystemVersions = "ListFilesystemVersions"
RPCReceive = "Receive"
RPCSend = "Send"
RPCReceive = "Receive"
RPCSend = "Send"
)
// Remote implements an endpoint stub that uses streamrpc as a transport.
@@ -295,13 +294,13 @@ func (s Remote) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Read
var res pdu.SendRes
if err := proto.Unmarshal(rb.Bytes(), &res); err != nil {
rs.Close()
return nil, nil, err
return nil, nil, err
}
// FIXME make sure the consumer will read the reader until the end...
return &res, rs, nil
}
func (s Remote) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) (error) {
func (s Remote) Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error {
defer sendStream.Close()
b, err := proto.Marshal(r)
if err != nil {
@@ -414,6 +413,6 @@ func (a *Handler) Handle(ctx context.Context, endpoint string, reqStructured *by
return bytes.NewBuffer(b), nil, err
}
Err:
Err:
return nil, nil, errors.New("no handler for given endpoint")
}
+3 -3
View File
@@ -43,7 +43,7 @@ func NewClientAuthListener(
}
tlsConf := tls.Config{
Certificates: []tls.Certificate{serverCert},
Certificates: []tls.Certificate{serverCert},
ClientCAs: ca,
ClientAuth: tls.RequireAndVerifyClientCert,
PreferServerCipherSuites: true,
@@ -114,8 +114,8 @@ func ClientAuthClient(serverName string, rootCA *x509.CertPool, clientCert tls.C
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{clientCert},
RootCAs: rootCA,
ServerName: serverName,
ServerName: serverName,
}
tlsConfig.BuildNameToCertificate()
return tlsConfig, nil
}
}