experiment: zync: an rsync-like tool for ZFS datasets (re-uses zrepl abstractions)

Local sync and sync over SSH seems to work. The sshdirect transport might be interesting / re-usable for zrepl as well, although we'd still need some kind of locking in that case.
2021-01-14 01:31:08 +01:00
parent 96d5288667
commit bd30afaaf9
8 changed files with 939 additions and 0 deletions
@@ -0,0 +1,406 @@
+package sshdirect
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/yamux"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/util/circlog"
+)
+
+type Endpoint struct {
+	Host         string
+	User         string
+	Port         uint16
+	IdentityFile string
+	SSHCommand   string
+	Options      []string
+	RunCommand   []string
+}
+
+func (e Endpoint) CmdArgs() (cmd string, args []string, env []string) {
+
+	if e.SSHCommand != "" {
+		cmd = e.SSHCommand
+	} else {
+		cmd = "ssh"
+	}
+
+	args = make([]string, 0, 2*len(e.Options)+4)
+	args = append(args,
+		"-p", fmt.Sprintf("%d", e.Port),
+		"-T",
+		"-i", e.IdentityFile,
+		"-o", "BatchMode=yes",
+	)
+	for _, option := range e.Options {
+		args = append(args, "-o", option)
+	}
+	args = append(args, fmt.Sprintf("%s@%s", e.User, e.Host))
+
+	args = append(args, e.RunCommand...)
+
+	env = []string{}
+
+	return
+}
+
+type SSHConn struct {
+	cmd    *exec.Cmd
+	stdin  io.WriteCloser
+	stdout io.ReadCloser
+
+	shutdownMtx    sync.Mutex
+	shutdownResult *shutdownResult // TODO not used anywhere
+	cmdCancel      context.CancelFunc
+}
+
+const go_network string = "netssh"
+
+type clientAddr struct {
+	pid int
+}
+
+func (a clientAddr) Network() string {
+	return go_network
+}
+
+func (a clientAddr) String() string {
+	return fmt.Sprintf("pid=%d", a.pid)
+}
+
+func (conn *SSHConn) LocalAddr() net.Addr {
+	proc := conn.cmd.Process
+	if proc == nil {
+		return clientAddr{-1}
+	}
+	return clientAddr{proc.Pid}
+}
+
+func (conn *SSHConn) RemoteAddr() net.Addr {
+	return conn.LocalAddr()
+}
+
+// Read implements io.Reader.
+// It returns *IOError for any non-nil error that is != io.EOF.
+func (conn *SSHConn) Read(p []byte) (int, error) {
+	n, err := conn.stdout.Read(p)
+	if err != nil && err != io.EOF {
+		return n, &IOError{err}
+	}
+	return n, err
+}
+
+// Write implements io.Writer.
+// It returns *IOError for any error != nil.
+func (conn *SSHConn) Write(p []byte) (int, error) {
+	n, err := conn.stdin.Write(p)
+	if err != nil {
+		return n, &IOError{err}
+	}
+	return n, err
+}
+
+func (conn *SSHConn) CloseWrite() error {
+	return conn.stdin.Close()
+}
+
+type deadliner interface {
+	SetReadDeadline(time.Time) error
+	SetWriteDeadline(time.Time) error
+}
+
+func (conn *SSHConn) SetReadDeadline(t time.Time) error {
+	// type assertion is covered by test TestExecCmdPipesDeadlineBehavior
+	return conn.stdout.(deadliner).SetReadDeadline(t)
+}
+
+func (conn *SSHConn) SetWriteDeadline(t time.Time) error {
+	// type assertion is covered by test TestExecCmdPipesDeadlineBehavior
+	return conn.stdin.(deadliner).SetWriteDeadline(t)
+}
+
+func (conn *SSHConn) SetDeadline(t time.Time) error {
+	// try both
+	rerr := conn.SetReadDeadline(t)
+	werr := conn.SetWriteDeadline(t)
+	if rerr != nil {
+		return rerr
+	}
+	if werr != nil {
+		return werr
+	}
+	return nil
+}
+
+func (conn *SSHConn) Close() error {
+	conn.shutdownProcess()
+	return nil // FIXME: waitError will be non-zero because we signaled it, shutdownProcess needs to distinguish that
+}
+
+type shutdownResult struct {
+	waitErr error
+}
+
+func (conn *SSHConn) shutdownProcess() *shutdownResult {
+	conn.shutdownMtx.Lock()
+	defer conn.shutdownMtx.Unlock()
+
+	if conn.shutdownResult != nil {
+		return conn.shutdownResult
+	}
+
+	wait := make(chan error, 1)
+	go func() {
+		if err := conn.cmd.Process.Signal(syscall.SIGTERM); err != nil {
+			// TODO log error
+			return
+		}
+		wait <- conn.cmd.Wait()
+	}()
+
+	timeout := time.NewTimer(1 * time.Second) // FIXME const
+	defer timeout.Stop()
+
+	select {
+	case waitErr := <-wait:
+		conn.shutdownResult = &shutdownResult{waitErr}
+	case <-timeout.C:
+		conn.cmdCancel()
+		waitErr := <-wait // reuse existing Wait invocation, must not call twice
+		conn.shutdownResult = &shutdownResult{waitErr}
+	}
+	return conn.shutdownResult
+}
+
+// Cmd returns the underlying *exec.Cmd (the ssh client process)
+// Use read-only, should not be necessary for regular users.
+func (conn *SSHConn) Cmd() *exec.Cmd {
+	return conn.cmd
+}
+
+// CmdCancel bypasses the normal shutdown mechanism of SSHConn
+// (that is, calling Close) and cancels the process's context,
+// which usually results in SIGKILL being sent to the process.
+// Intended for integration tests, regular users shouldn't use it.
+func (conn *SSHConn) CmdCancel() {
+	conn.cmdCancel()
+}
+
+const bannerMessageLen = 31
+
+var messages = make(map[string][]byte)
+
+func mustMessage(str string) []byte {
+	if len(str) > bannerMessageLen {
+		panic("message length must be smaller than bannerMessageLen")
+	}
+	if _, ok := messages[str]; ok {
+		panic("duplicate message")
+	}
+	var buf bytes.Buffer
+	n, _ := buf.WriteString(str)
+	if n != len(str) {
+		panic("message must only contain ascii / 8-bit chars")
+	}
+	buf.Write(bytes.Repeat([]byte{0}, bannerMessageLen-n))
+	return buf.Bytes()
+}
+
+var banner_msg = mustMessage("SSDIRECTHCON_HELO")
+var proxy_error_msg = mustMessage("SSDIRECTHCON_PROXY_ERROR") /* FIXME irrelevant, was copy-pasta */
+var begin_msg = mustMessage("SSDIRECTHCON_BEGIN")
+
+type SSHError struct {
+	RWCError      error
+	WhileActivity string
+}
+
+// Error() will try to present a one-line error message unless ssh stderr output is longer than one line
+func (e *SSHError) Error() string {
+
+	exitErr, ok := e.RWCError.(*exec.ExitError)
+	if !ok {
+		return fmt.Sprintf("ssh: %s", e.RWCError)
+	}
+
+	ws := exitErr.ProcessState.Sys().(syscall.WaitStatus)
+	var wsmsg string
+	if ws.Exited() {
+		wsmsg = fmt.Sprintf("(exit status %d)", ws.ExitStatus())
+	} else {
+		wsmsg = fmt.Sprintf("(%s)", ws.Signal())
+	}
+
+	haveSSHMessage := len(exitErr.Stderr) > 0
+	sshOnelineStderr := false
+	if i := bytes.Index(exitErr.Stderr, []byte("\n")); i == len(exitErr.Stderr)-1 {
+		sshOnelineStderr = true
+	}
+	stderr := bytes.TrimSpace(exitErr.Stderr)
+
+	if haveSSHMessage {
+		if sshOnelineStderr {
+			return fmt.Sprintf("ssh: '%s' %s", stderr, wsmsg) // FIXME proper single-quoting
+		} else {
+			return fmt.Sprintf("ssh %s\n%s", wsmsg, stderr)
+		}
+	}
+
+	return fmt.Sprintf("ssh terminated without stderr output %s", wsmsg)
+
+}
+
+type ProtocolError struct {
+	What string
+}
+
+func (e ProtocolError) Error() string {
+	return e.What
+}
+
+// Dial connects to the remote endpoint where it expects a command executing Proxy().
+// Dial performs a handshake consisting of the exchange of banner messages before returning the connection.
+// If the handshake cannot be completed before dialCtx is Done(), the underlying ssh command is killed
+// and the dialCtx.Err() returned.
+// If the handshake completes, dialCtx's deadline does not affect the returned connection.
+//
+// Errors returned are either dialCtx.Err(), or intances of ProtocolError or *SSHError
+func Dial(dialCtx context.Context, endpoint Endpoint) (*SSHConn, error) {
+
+	sshCmd, sshArgs, sshEnv := endpoint.CmdArgs()
+	commandCtx, commandCancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(commandCtx, sshCmd, sshArgs...)
+	cmd.Env = sshEnv
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stderrBuf, err := circlog.NewCircularLog(1 << 15)
+	if err != nil {
+		panic(err) // wrong API usage
+	}
+	cmd.Stderr = stderrBuf
+
+	if err = cmd.Start(); err != nil {
+		return nil, err
+	}
+	cmdWaitErrOrIOErr := func(ioErr error, what string) *SSHError {
+		werr := cmd.Wait()
+		if werr, ok := werr.(*exec.ExitError); ok {
+			werr.Stderr = []byte(stderrBuf.String())
+			return &SSHError{werr, what}
+		}
+		return &SSHError{ioErr, what}
+	}
+
+	confErrChan := make(chan error, 1)
+	go func() {
+		defer close(confErrChan)
+		var buf bytes.Buffer
+		if _, err := io.CopyN(&buf, stdout, int64(len(banner_msg))); err != nil {
+			confErrChan <- cmdWaitErrOrIOErr(err, "read banner")
+			return
+		}
+		resp := buf.Bytes()
+		switch {
+		case bytes.Equal(resp, banner_msg):
+			break
+		case bytes.Equal(resp, proxy_error_msg):
+			_ = cmdWaitErrOrIOErr(nil, "")
+			confErrChan <- ProtocolError{"proxy error, check remote configuration"}
+			return
+		default:
+			_ = cmdWaitErrOrIOErr(nil, "")
+			confErrChan <- ProtocolError{fmt.Sprintf("unknown banner message: %v", resp)}
+			return
+		}
+		buf.Reset()
+		buf.Write(begin_msg)
+		if _, err := io.Copy(stdin, &buf); err != nil {
+			confErrChan <- cmdWaitErrOrIOErr(err, "send begin message")
+			return
+		}
+	}()
+
+	select {
+	case <-dialCtx.Done():
+
+		commandCancel()
+		// cancelling will make one of the calls in above goroutine fail,
+		// and the goroutine will send the error to confErrChan
+		//
+		// ignore the error and return the cancellation cause
+
+		// draining always terminates because we know the channel is always closed
+		for _ = range confErrChan {
+		}
+
+		// TODO collect stderr in this case
+		// can probably extend *SSHError for this but need to implement net.Error
+
+		return nil, dialCtx.Err()
+
+	case err := <-confErrChan:
+		if err != nil {
+			commandCancel()
+			return nil, err
+		}
+	}
+
+	return &SSHConn{
+		cmd:       cmd,
+		stdin:     stdin,
+		stdout:    stdout,
+		cmdCancel: commandCancel,
+	}, nil
+}
+
+type Connecter struct {
+	s        *yamux.Session
+	endpoint Endpoint
+}
+
+var _ transport.Connecter = (*Connecter)(nil)
+
+func NewConnecter(ctx context.Context, endpoint Endpoint) (*Connecter, error) {
+	conn, err := Dial(ctx, endpoint)
+	if err != nil {
+		return nil, err
+	}
+	s, err := yamux.Client(conn, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &Connecter{
+		s:        s,
+		endpoint: endpoint,
+	}, nil
+}
+
+type fakeWire struct {
+	net.Conn
+}
+
+func (w *fakeWire) CloseWrite() error {
+	time.Sleep(1*time.Second) // HACKY
+	return fmt.Errorf("fakeWire does not support CloseWrite")
+}
+
+func (c *Connecter) Connect(ctx context.Context) (transport.Wire, error) {
+	conn, err := c.s.Open()
+	return &fakeWire{conn}, err
+}
@@ -0,0 +1,48 @@
+package sshdirect
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"syscall"
+)
+
+type timeouter interface {
+	Timeout() bool
+}
+
+var _ timeouter = &os.PathError{}
+
+type IOError struct {
+	Cause error
+}
+
+var _ net.Error = &IOError{}
+
+func (e IOError) GoString() string {
+	return fmt.Sprintf("ServeConnIOError:%#v", e.Cause)
+}
+
+func (e IOError) Error() string {
+	// following case found by experiment
+	if pathErr, ok := e.Cause.(*os.PathError); ok {
+		if pathErr.Err == syscall.EPIPE {
+			return fmt.Sprintf("netssh %s: %s (likely: connection reset by peer)",
+				pathErr.Op, pathErr.Err,
+			)
+		}
+		return fmt.Sprintf("netssh: %s: %s", pathErr.Op, pathErr.Err)
+	}
+	return fmt.Sprintf("netssh: %s", e.Cause.Error())
+}
+
+func (e IOError) Timeout() bool {
+	if to, ok := e.Cause.(timeouter); ok {
+		return to.Timeout()
+	}
+	return false
+}
+
+func (e IOError) Temporary() bool {
+	return false
+}
@@ -0,0 +1,91 @@
+package sshdirect
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"net"
+	"os"
+	"time"
+
+	"github.com/hashicorp/yamux"
+)
+
+type ServeConn struct {
+	stdin, stdout *os.File
+}
+
+var _ net.Conn = (*ServeConn)(nil)
+
+func ServeStdin() (net.Listener, error) {
+
+	conn := &ServeConn{
+		stdin:  os.Stdin,
+		stdout: os.Stdout,
+	}
+
+	var buf bytes.Buffer
+	buf.Write(banner_msg)
+	if _, err := io.Copy(conn, &buf); err != nil {
+		log.Printf("error sending confirm message: %s", err)
+		conn.Close()
+		return nil, err
+	}
+	buf.Reset()
+	if _, err := io.CopyN(&buf, conn, int64(len(begin_msg))); err != nil {
+		log.Printf("error reading begin message: %s", err)
+		conn.Close()
+		return nil, err
+	}
+
+	return yamux.Server(conn, nil)
+}
+
+func (c *ServeConn) Read(p []byte) (int, error) {
+	return c.stdin.Read(p)
+}
+
+func (c *ServeConn) Write(p []byte) (int, error) {
+	return c.stdout.Write(p)
+}
+
+func (f *ServeConn) Close() (err error) {
+	e1 := f.stdin.Close()
+	e2 := f.stdout.Close()
+	// FIXME merge errors
+	if e1 != nil {
+		return e1
+	}
+	return e2
+}
+
+func (f *ServeConn) SetReadDeadline(t time.Time) error {
+	return f.stdin.SetReadDeadline(t)
+}
+
+func (f *ServeConn) SetWriteDeadline(t time.Time) error {
+	return f.stdout.SetReadDeadline(t)
+}
+
+func (f *ServeConn) SetDeadline(t time.Time) error {
+	// try both...
+	werr := f.SetWriteDeadline(t)
+	rerr := f.SetReadDeadline(t)
+	if werr != nil {
+		return werr
+	}
+	if rerr != nil {
+		return rerr
+	}
+	return nil
+}
+
+type serveAddr struct{}
+
+const GoNetwork string = "sshdirect"
+
+func (serveAddr) Network() string { return GoNetwork }
+func (serveAddr) String() string  { return "???" }
+
+func (f *ServeConn) LocalAddr() net.Addr  { return serveAddr{} }
+func (f *ServeConn) RemoteAddr() net.Addr { return serveAddr{} }