rpc rewrite: control RPCs using gRPC + separate RPC for data transfer

transport/ssh: update go-netssh to new version => supports CloseWrite and Deadlines => build: require Go 1.11 (netssh requires it)
2018-12-11 22:01:50 +01:00
parent d281fb00e3
commit 796c5ad42d
100 changed files with 6460 additions and 1485 deletions
@@ -0,0 +1,12 @@
+# setup-specific
+inventory
+*.retry
+
+# generated by gen_files.sh
+files/*ssh_client_identity
+files/*ssh_client_identity.pub
+files/*.tls.*.key
+files/*.tls.*.csr
+files/*.tls.*.crt
+files/wireevaluator
+
@@ -0,0 +1,15 @@
+This directory contains very hacky test automation for wireevaluator based on nested Ansible playbooks.
+
+* Copy `inventory.example` to `inventory`
+* Adjust `inventory` IP addresses as needed
+* Make sure there's an OpenSSH server running on the serve host
+* Make sure there's no firewalling whatsoever between the hosts
+* Run `GENKEYS=1 ./gen_files.sh` to re-generate self-signed TLS certs
+* Run the following command, adjusting the `wireevaluator_repeat` value to the number of times you want to repeat each test
+
+```
+ansible-playbook -i inventory all.yml -e `wireevaluator_repeat=3`
+```
+
+Generally, things are fine if the playbook doesn't show any panics from wireevaluator.
+
@@ -0,0 +1,17 @@
+- hosts: connect,serve
+  tasks:
+    
+    - name: "run test"
+      include: internal_prepare_and_run_repeated.yml
+      wireevaluator_transport: "{{config.0}}"
+      wireevaluator_case: "{{config.1}}"
+      wireevaluator_repeat: "{{wireevaluator_repeat}}"
+      with_cartesian:
+        - [ tls, ssh, tcp ]
+        -
+          - closewrite_server
+          - closewrite_client
+          - readdeadline_server
+          - readdeadline_client
+      loop_control:
+        loop_var: config
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$( dirname "${BASH_SOURCE[0]}")"
+
+FILESDIR="$(pwd)"/files
+
+echo "[INFO] compile binary"
+pushd .. >/dev/null
+go build -o $FILESDIR/wireevaluator
+popd >/dev/null
+
+if [ "$GENKEYS" == "" ]; then
+    echo "[INFO] GENKEYS environment variable not set, assumed to be valid"
+    exit 0
+fi
+
+echo "[INFO] gen ssh key"
+ssh-keygen -f "$FILESDIR/wireevaluator.ssh_client_identity" -t ed25519
+
+echo "[INFO] gen tls keys"
+
+cakey="$FILESDIR/wireevaluator.tls.ca.key"
+cacrt="$FILESDIR/wireevaluator.tls.ca.crt"
+hostprefix="$FILESDIR/wireevaluator.tls"
+
+openssl genrsa -out "$cakey" 4096
+openssl req -x509 -new -nodes -key "$cakey" -sha256 -days 1 -out "$cacrt"
+
+declare -a HOSTS
+HOSTS+=("theserver")
+HOSTS+=("theclient")
+
+for host in "${HOSTS[@]}"; do
+    key="${hostprefix}.${host}.key"
+    csr="${hostprefix}.${host}.csr"
+    crt="${hostprefix}.${host}.crt"
+    openssl genrsa -out "$key" 2048
+
+    (
+        echo "."
+        echo "."
+        echo "."
+        echo "."
+        echo "."
+        echo $host
+        echo "."
+        echo "."
+        echo "."
+        echo "."
+    ) | openssl req -new -key "$key" -out "$csr"
+
+    openssl x509 -req -in "$csr" -CA "$cacrt" -CAkey "$cakey" -CAcreateserial -out "$crt" -days 1 -sha256
+
+done
@@ -0,0 +1,54 @@
+---
+
+- name: compile binary and any key files required
+  local_action: command ./gen_files.sh
+
+- name: Kill test binary
+  shell: "killall -9 wireevaluator || true"
+- name: Deploy new binary
+  copy:
+    src: "files/wireevaluator"
+    dest: "/opt/wireevaluator"
+    mode: 0755
+
+- set_fact:
+    wireevaluator_connect_ip: "{{hostvars['connect'].ansible_host}}"
+    wireevaluator_serve_ip: "{{hostvars['serve'].ansible_host}}"
+
+- name: Deploy config
+  template:
+    src: "templates/{{wireevaluator_transport}}.yml.j2"
+    dest: "/opt/wireevaluator.yml"
+
+- name: Deploy client identity
+  copy:
+    src: "files/wireevaluator.{{item}}"
+    dest: "/opt/wireevaluator.{{item}}"
+    mode: 0400
+  with_items:
+    - ssh_client_identity
+    - ssh_client_identity.pub
+    - tls.ca.key
+    - tls.ca.crt
+    - tls.theserver.key
+    - tls.theserver.crt
+    - tls.theclient.key
+    - tls.theclient.crt
+
+- name: Setup server ssh client identity access
+  when: inventory_hostname == "serve"
+  block:
+  - authorized_key:
+      user: root
+      state: present
+      key: "{{ lookup('file', 'files/wireevaluator.ssh_client_identity.pub') }}"
+      key_options: 'command="/opt/wireevaluator -mode stdinserver -config /opt/wireevaluator.yml client1"'
+  - file:
+      state: directory
+      mode: 0700
+      path: /tmp/wireevaluator_stdinserver
+
+- name: repeated test 
+  include: internal_run_test_prepared_single.yml
+  with_sequence: start=1 end={{wireevaluator_repeat}}
+
@@ -0,0 +1,38 @@
+---
+
+- debug:
+    msg: "run test transport={{wireevaluator_transport}} case={{wireevaluator_case}} repeatedly"
+
+- name: Run Server
+  when: inventory_hostname == "serve"
+  command: /opt/wireevaluator -config /opt/wireevaluator.yml -mode serve -testcase {{wireevaluator_case}}
+  register: spawn_servers
+  async: 60
+  poll: 0
+
+- name: Run Client
+  when: inventory_hostname == "connect"
+  command: /opt/wireevaluator -config /opt/wireevaluator.yml -mode connect -testcase {{wireevaluator_case}}
+  register: spawn_clients
+  async: 60
+  poll: 0
+
+- name: Wait for server shutdown
+  when: inventory_hostname == "serve"
+  async_status:
+    jid: "{{ spawn_servers.ansible_job_id}}"
+  delay: 0.5
+  retries: 10
+
+- name: Wait for client shutdown
+  when: inventory_hostname == "connect"
+  async_status:
+    jid: "{{ spawn_clients.ansible_job_id}}"
+  delay: 0.5
+  retries: 10
+
+- name: Wait for connections to die (TIME_WAIT conns)
+  command: sleep 4
+  changed_when: false
+
+
@@ -0,0 +1,2 @@
+connect ansible_user=root ansible_host=192.168.122.128 wireevaluator_mode="connect"
+serve   ansible_user=root ansible_host=192.168.122.129 wireevaluator_mode="serve"
@@ -0,0 +1,13 @@
+connect:
+  type: ssh+stdinserver
+  host: {{wireevaluator_serve_ip}}
+  user: root
+  port: 22
+  identity_file: /opt/wireevaluator.ssh_client_identity
+  options: # optional, default [], `-o` arguments passed to ssh
+  - "Compression=yes"
+serve:
+  type: stdinserver
+  client_identities:
+    - "client1"
+
@@ -0,0 +1,10 @@
+connect:
+    type: tcp
+    address: "{{wireevaluator_serve_ip}}:8888"
+serve:
+    type: tcp
+    listen: ":8888"
+    clients: {
+        "{{wireevaluator_connect_ip}}" : "client1"
+    }
+
@@ -0,0 +1,16 @@
+connect:
+  type: tls
+  address: "{{wireevaluator_serve_ip}}:8888"
+  ca: "/opt/wireevaluator.tls.ca.crt"
+  cert: "/opt/wireevaluator.tls.theclient.crt"
+  key: "/opt/wireevaluator.tls.theclient.key"
+  server_cn: "theserver"
+
+serve:
+  type: tls
+  listen: ":8888"
+  ca: "/opt/wireevaluator.tls.ca.crt"
+  cert: "/opt/wireevaluator.tls.theserver.crt"
+  key: "/opt/wireevaluator.tls.theserver.key"
+  client_cns:
+    - "theclient"
@@ -0,0 +1,111 @@
+// a tool to test whether a given transport implements the timeoutconn.Wire interface
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	netssh "github.com/problame/go-netssh"
+	"github.com/zrepl/yaml-config"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/transport"
+	transportconfig "github.com/zrepl/zrepl/transport/fromconfig"
+)
+
+func noerror(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Config struct {
+	Connect config.ConnectEnum
+	Serve   config.ServeEnum
+}
+
+var args struct {
+	mode       string
+	configPath string
+	testCase   string
+}
+
+var conf Config
+
+type TestCase interface {
+	Client(wire transport.Wire)
+	Server(wire transport.Wire)
+}
+
+func main() {
+	flag.StringVar(&args.mode, "mode", "", "connect|serve")
+	flag.StringVar(&args.configPath, "config", "", "config file path")
+	flag.StringVar(&args.testCase, "testcase", "", "")
+	flag.Parse()
+
+	bytes, err := ioutil.ReadFile(args.configPath)
+	noerror(err)
+	err = yaml.UnmarshalStrict(bytes, &conf)
+	noerror(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	global := &config.Global{
+		Serve: &config.GlobalServe{
+			StdinServer: &config.GlobalStdinServer{
+				SockDir: "/tmp/wireevaluator_stdinserver",
+			},
+		},
+	}
+
+	switch args.mode {
+	case "connect":
+		tc, err := getTestCase(args.testCase)
+		noerror(err)
+		connecter, err := transportconfig.ConnecterFromConfig(global, conf.Connect)
+		noerror(err)
+		wire, err := connecter.Connect(ctx)
+		noerror(err)
+		tc.Client(wire)
+	case "serve":
+		tc, err := getTestCase(args.testCase)
+		noerror(err)
+		lf, err := transportconfig.ListenerFactoryFromConfig(global, conf.Serve)
+		noerror(err)
+		l, err := lf()
+		noerror(err)
+		conn, err := l.Accept(ctx)
+		noerror(err)
+		tc.Server(conn)
+	case "stdinserver":
+		identity := flag.Arg(0)
+		unixaddr := path.Join(global.Serve.StdinServer.SockDir, identity)
+		err := netssh.Proxy(ctx, unixaddr)
+		if err == nil {
+			os.Exit(0)
+		}
+		panic(err)
+	default:
+		panic(fmt.Sprintf("unknown mode %q", args.mode))
+	}
+
+}
+
+func getTestCase(tcName string) (TestCase, error) {
+	switch tcName {
+	case "closewrite_server":
+		return &CloseWrite{mode: CloseWriteServerSide}, nil
+	case "closewrite_client":
+		return &CloseWrite{mode: CloseWriteClientSide}, nil
+	case "readdeadline_client":
+		return &Deadlines{mode: DeadlineModeClientTimeout}, nil
+	case "readdeadline_server":
+		return &Deadlines{mode: DeadlineModeServerTimeout}, nil
+	default:
+		return nil, fmt.Errorf("unknown test case %q", tcName)
+	}
+}
@@ -0,0 +1,110 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"log"
+
+	"github.com/zrepl/zrepl/transport"
+)
+
+type CloseWriteMode uint
+
+const (
+	CloseWriteClientSide CloseWriteMode = 1 + iota
+	CloseWriteServerSide
+)
+
+type CloseWrite struct {
+	mode CloseWriteMode
+}
+
+// sent repeatedly
+var closeWriteTestSendData = bytes.Repeat([]byte{0x23, 0x42}, 1<<24)
+var closeWriteErrorMsg = []byte{0xb, 0xa, 0xd, 0xf, 0x0, 0x0, 0xd}
+
+func (m CloseWrite) Client(wire transport.Wire) {
+	switch m.mode {
+	case CloseWriteClientSide:
+		m.receiver(wire)
+	case CloseWriteServerSide:
+		m.sender(wire)
+	default:
+		panic(m.mode)
+	}
+}
+
+func (m CloseWrite) Server(wire transport.Wire) {
+	switch m.mode {
+	case CloseWriteClientSide:
+		m.sender(wire)
+	case CloseWriteServerSide:
+		m.receiver(wire)
+	default:
+		panic(m.mode)
+	}
+}
+
+func (CloseWrite) sender(wire transport.Wire) {
+	defer func() {
+		closeErr := wire.Close()
+		log.Printf("closeErr=%T %s", closeErr, closeErr)
+	}()
+
+	type opResult struct {
+		err error
+	}
+	writeDone := make(chan struct{}, 1)
+	go func() {
+		close(writeDone)
+		for {
+			_, err := wire.Write(closeWriteTestSendData)
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	defer func() {
+		<-writeDone
+	}()
+
+	var respBuf bytes.Buffer
+	_, err := io.Copy(&respBuf, wire)
+	if err != nil {
+		log.Fatalf("should have received io.EOF, which is masked by io.Copy, got: %s", err)
+	}
+	if !bytes.Equal(respBuf.Bytes(), closeWriteErrorMsg) {
+		log.Fatalf("did not receive error message, got response with len %v:\n%v", respBuf.Len(), respBuf.Bytes())
+	}
+
+}
+
+func (CloseWrite) receiver(wire transport.Wire) {
+
+	// consume half the test data, then detect an error, send it and CloseWrite
+
+	r := io.LimitReader(wire, int64(5 * len(closeWriteTestSendData)/3))
+	_, err := io.Copy(ioutil.Discard, r)
+	noerror(err)
+
+	var errBuf bytes.Buffer
+	errBuf.Write(closeWriteErrorMsg)
+	_, err = io.Copy(wire, &errBuf)
+	noerror(err)
+
+	err = wire.CloseWrite()
+	noerror(err)
+
+	// drain wire, as documented in transport.Wire, this is the only way we know the client closed the conn
+	_, err = io.Copy(ioutil.Discard, wire)
+	if err != nil {
+		// io.Copy masks io.EOF to nil, and we expect io.EOF from the client's Close() call
+		log.Panicf("unexpected error returned from reading conn: %s", err)
+	}
+		
+	closeErr := wire.Close()
+	log.Printf("closeErr=%T %s", closeErr, closeErr)
+
+}
@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/zrepl/zrepl/transport"
+)
+
+type DeadlineMode uint
+
+const (
+	DeadlineModeClientTimeout DeadlineMode = 1 + iota
+	DeadlineModeServerTimeout
+)
+
+type Deadlines struct {
+	mode DeadlineMode
+}
+
+func (d Deadlines) Client(wire transport.Wire) {
+	switch d.mode {
+	case DeadlineModeClientTimeout:
+		d.sleepThenSend(wire)
+	case DeadlineModeServerTimeout:
+		d.sendThenRead(wire)
+	default:
+		panic(d.mode)
+	}
+}
+
+func (d Deadlines) Server(wire transport.Wire) {
+	switch d.mode {
+	case DeadlineModeClientTimeout:
+		d.sendThenRead(wire)
+	case DeadlineModeServerTimeout:
+		d.sleepThenSend(wire)
+	default:
+		panic(d.mode)
+	}
+}
+
+var deadlinesTimeout = 1 * time.Second
+
+func (d Deadlines) sleepThenSend(wire transport.Wire) {
+	defer wire.Close()
+
+	log.Print("sleepThenSend")
+
+	// exceed timeout of peer (do not respond to their hi msg)
+	time.Sleep(3 * deadlinesTimeout)
+	// expect that the client has hung up on us by now
+	err := d.sendMsg(wire, "hi")
+	log.Printf("err=%s", err)
+	log.Printf("err=%#v", err)
+	if err == nil {
+		log.Panic("no error")
+	}
+	if _, ok := err.(net.Error); !ok {
+		log.Panic("not a net error")
+	}
+
+}
+
+func (d Deadlines) sendThenRead(wire transport.Wire) {
+
+	log.Print("sendThenRead")
+
+	err := d.sendMsg(wire, "hi")
+	noerror(err)
+
+	err = wire.SetReadDeadline(time.Now().Add(deadlinesTimeout))
+	noerror(err)
+
+	m, err := d.recvMsg(wire)
+	log.Printf("m=%q", m)
+	log.Printf("err=%s", err)
+	log.Printf("err=%#v", err)
+
+	// close asap so that the peer get's a 'connection reset by peer' error or similar
+	closeErr := wire.Close()
+	if closeErr != nil {
+		panic(closeErr)
+	}
+
+	var neterr net.Error
+	var ok bool
+	if err == nil {
+		goto unexpErr // works for nil, too
+	}
+	neterr, ok = err.(net.Error)
+	if !ok {
+		log.Println("not a net error")
+		goto unexpErr
+	}
+	if !neterr.Timeout() {
+		log.Println("not a timeout")
+	}
+
+	return
+
+unexpErr:
+	panic(fmt.Sprintf("sendThenRead: client should have hung up but got error %T %s", err, err))
+}
+
+const deadlinesMsgLen = 40
+
+func (d Deadlines) sendMsg(wire transport.Wire, msg string) error {
+	if len(msg) > deadlinesMsgLen {
+		panic(len(msg))
+	}
+	var buf [deadlinesMsgLen]byte
+	copy(buf[:], []byte(msg))
+	n, err := wire.Write(buf[:])
+	if err != nil {
+		return err
+	}
+	if n != len(buf) {
+		panic("short write not allowed")
+	}
+	return nil
+}
+
+func (d Deadlines) recvMsg(wire transport.Wire) (string, error) {
+
+	var buf bytes.Buffer
+	r := io.LimitReader(wire, deadlinesMsgLen)
+	_, err := io.Copy(&buf, r)
+	if err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+
+}