S3 loggyloo (#511)

* add minio-go dep, update deps

* add minio s3 client

minio has an s3 compatible api and is an open source project and, notably, is
not amazon, so it seems best to use their client (fwiw the aws-sdk-go is a
giant hair ball of things we don't need, too). it was pretty easy and seems
to work, so rolling with it. also, minio is a totally feasible option for fn
installs in prod / for demos / for local.

* adds 's3' package for s3 compatible log storage api, for use with storing
logs from calls and retrieving them.
* removes DELETE /v1/apps/:app/calls/:call/log endpoint
* removes internal log deletion api
* changes the GetLog API to use an io.Reader, which is a backwards step atm
due to the json api for logs, I have another branch lined up to make a plain
text log API and this will be much more efficient (also want to gzip)
* hooked up minio to the test suite and fixed up the test suite
* add how to run minio docs and point fn at it docs

some notes: notably we aren't cleaning up these logs. there is a ticket
already to make a Mr. Clean who wakes up periodically and nukes old stuff, so
am punting any api design around some kind of TTL deletion of logs. there are
a lot of options really for Mr. Clean, we can notably defer to him when apps
are deleted, too, so that app deletion is fast and then Mr. Clean will just
clean them up later (seems like a good option).

have not tested against BMC object store, which has an s3 compatible API. but
in theory it 'just works' (the reason for doing this). in any event, that's
part of the service land to figure out.

closes #481
closes #473

* add log not found error to minio land

vendor/golang.org/x/crypto/ssh/cipher.go

@@ -304,7 +304,7 @@ type gcmCipher struct {
 	buf    []byte
 }
 
-func newGCMCipher(iv, key, macKey []byte) (packetCipher, error) {
+func newGCMCipher(iv, key []byte) (packetCipher, error) {
 	c, err := aes.NewCipher(key)
 	if err != nil {
 		return nil, err

vendor/golang.org/x/crypto/ssh/client.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"sync"
 	"time"
 )
@@ -187,6 +188,10 @@ func Dial(network, addr string, config *ClientConfig) (*Client, error) {
 // net.Conn underlying the the SSH connection.
 type HostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error
 
+// BannerCallback is the function type used for treat the banner sent by
+// the server. A BannerCallback receives the message sent by the remote server.
+type BannerCallback func(message string) error
+
 // A ClientConfig structure is used to configure a Client. It must not be
 // modified after having been passed to an SSH function.
 type ClientConfig struct {
@@ -209,6 +214,12 @@ type ClientConfig struct {
 	// FixedHostKey can be used for simplistic host key checks.
 	HostKeyCallback HostKeyCallback
 
+	// BannerCallback is called during the SSH dance to display a custom
+	// server's message. The client configuration can supply this callback to
+	// handle it as wished. The function BannerDisplayStderr can be used for
+	// simplistic display on Stderr.
+	BannerCallback BannerCallback
+
 	// ClientVersion contains the version identification string that will
 	// be used for the connection. If empty, a reasonable default is used.
 	ClientVersion string
@@ -255,3 +266,13 @@ func FixedHostKey(key PublicKey) HostKeyCallback {
 	hk := &fixedHostKey{key}
 	return hk.check
 }
+
+// BannerDisplayStderr returns a function that can be used for
+// ClientConfig.BannerCallback to display banners on os.Stderr.
+func BannerDisplayStderr() BannerCallback {
+	return func(banner string) error {
+		_, err := os.Stderr.WriteString(banner)
+
+		return err
+	}
+}

vendor/golang.org/x/crypto/ssh/client_auth.go

@@ -283,7 +283,9 @@ func confirmKeyAck(key PublicKey, c packetConn) (bool, error) {
 		}
 		switch packet[0] {
 		case msgUserAuthBanner:
-			// TODO(gpaul): add callback to present the banner to the user
+			if err := handleBannerResponse(c, packet); err != nil {
+				return false, err
+			}
 		case msgUserAuthPubKeyOk:
 			var msg userAuthPubKeyOkMsg
 			if err := Unmarshal(packet, &msg); err != nil {
@@ -325,7 +327,9 @@ func handleAuthResponse(c packetConn) (bool, []string, error) {
 
 		switch packet[0] {
 		case msgUserAuthBanner:
-			// TODO: add callback to present the banner to the user
+			if err := handleBannerResponse(c, packet); err != nil {
+				return false, nil, err
+			}
 		case msgUserAuthFailure:
 			var msg userAuthFailureMsg
 			if err := Unmarshal(packet, &msg); err != nil {
@@ -340,6 +344,24 @@ func handleAuthResponse(c packetConn) (bool, []string, error) {
 	}
 }
 
+func handleBannerResponse(c packetConn, packet []byte) error {
+	var msg userAuthBannerMsg
+	if err := Unmarshal(packet, &msg); err != nil {
+		return err
+	}
+
+	transport, ok := c.(*handshakeTransport)
+	if !ok {
+		return nil
+	}
+
+	if transport.bannerCallback != nil {
+		return transport.bannerCallback(msg.Message)
+	}
+
+	return nil
+}
+
 // KeyboardInteractiveChallenge should print questions, optionally
 // disabling echoing (e.g. for passwords), and return all the answers.
 // Challenge may be called multiple times in a single session. After
@@ -385,7 +407,9 @@ func (cb KeyboardInteractiveChallenge) auth(session []byte, user string, c packe
 		// like handleAuthResponse, but with less options.
 		switch packet[0] {
 		case msgUserAuthBanner:
-			// TODO: Print banners during userauth.
+			if err := handleBannerResponse(c, packet); err != nil {
+				return false, nil, err
+			}
 			continue
 		case msgUserAuthInfoRequest:
 			// OK

vendor/golang.org/x/crypto/ssh/client_test.go

@@ -79,3 +79,41 @@ func TestHostKeyCheck(t *testing.T) {
 		}
 	}
 }
+
+func TestBannerCallback(t *testing.T) {
+	c1, c2, err := netPipe()
+	if err != nil {
+		t.Fatalf("netPipe: %v", err)
+	}
+	defer c1.Close()
+	defer c2.Close()
+
+	serverConf := &ServerConfig{
+		NoClientAuth: true,
+		BannerCallback: func(conn ConnMetadata) string {
+			return "Hello World"
+		},
+	}
+	serverConf.AddHostKey(testSigners["rsa"])
+	go NewServerConn(c1, serverConf)
+
+	var receivedBanner string
+	clientConf := ClientConfig{
+		User:            "user",
+		HostKeyCallback: InsecureIgnoreHostKey(),
+		BannerCallback: func(message string) error {
+			receivedBanner = message
+			return nil
+		},
+	}
+
+	_, _, _, err = NewClientConn(c2, "", &clientConf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := "Hello World"
+	if receivedBanner != expected {
+		t.Fatalf("got %s; want %s", receivedBanner, expected)
+	}
+}

vendor/golang.org/x/crypto/ssh/handshake.go

@@ -78,6 +78,11 @@ type handshakeTransport struct {
 	dialAddress     string
 	remoteAddr      net.Addr
 
+	// bannerCallback is non-empty if we are the client and it has been set in
+	// ClientConfig. In that case it is called during the user authentication
+	// dance to handle a custom server's message.
+	bannerCallback BannerCallback
+
 	// Algorithms agreed in the last key exchange.
 	algorithms *algorithms
 
@@ -120,6 +125,7 @@ func newClientTransport(conn keyingTransport, clientVersion, serverVersion []byt
 	t.dialAddress = dialAddr
 	t.remoteAddr = addr
 	t.hostKeyCallback = config.HostKeyCallback
+	t.bannerCallback = config.BannerCallback
 	if config.HostKeyAlgorithms != nil {
 		t.hostKeyAlgorithms = config.HostKeyAlgorithms
 	} else {

vendor/golang.org/x/crypto/ssh/messages.go

@@ -23,10 +23,6 @@ const (
 	msgUnimplemented = 3
 	msgDebug         = 4
 	msgNewKeys       = 21
-
-	// Standard authentication messages
-	msgUserAuthSuccess = 52
-	msgUserAuthBanner  = 53
 )
 
 // SSH messages:
@@ -137,6 +133,18 @@ type userAuthFailureMsg struct {
 	PartialSuccess bool
 }
 
+// See RFC 4252, section 5.1
+const msgUserAuthSuccess = 52
+
+// See RFC 4252, section 5.4
+const msgUserAuthBanner = 53
+
+type userAuthBannerMsg struct {
+	Message string `sshtype:"53"`
+	// unused, but required to allow message parsing
+	Language string
+}
+
 // See RFC 4256, section 3.2
 const msgUserAuthInfoRequest = 60
 const msgUserAuthInfoResponse = 61

vendor/golang.org/x/crypto/ssh/server.go

@@ -95,6 +95,10 @@ type ServerConfig struct {
 	// Note that RFC 4253 section 4.2 requires that this string start with
 	// "SSH-2.0-".
 	ServerVersion string
+
+	// BannerCallback, if present, is called and the return string is sent to
+	// the client after key exchange completed but before authentication.
+	BannerCallback func(conn ConnMetadata) string
 }
 
 // AddHostKey adds a private key as a host key. If an existing host
@@ -343,6 +347,19 @@ userAuthLoop:
 		}
 
 		s.user = userAuthReq.User
+
+		if authFailures == 0 && config.BannerCallback != nil {
+			msg := config.BannerCallback(s)
+			if msg != "" {
+				bannerMsg := &userAuthBannerMsg{
+					Message: msg,
+				}
+				if err := s.transport.writePacket(Marshal(bannerMsg)); err != nil {
+					return nil, err
+				}
+			}
+		}
+
 		perms = nil
 		authErr := errors.New("no auth passed yet")
 

vendor/golang.org/x/crypto/ssh/terminal/util.go

@@ -17,44 +17,41 @@
 package terminal // import "golang.org/x/crypto/ssh/terminal"
 
 import (
-	"syscall"
-	"unsafe"
-
 	"golang.org/x/sys/unix"
 )
 
 // State contains the state of a terminal.
 type State struct {
-	termios syscall.Termios
+	termios unix.Termios
 }
 
 // IsTerminal returns true if the given file descriptor is a terminal.
 func IsTerminal(fd int) bool {
-	var termios syscall.Termios
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
-	return err == 0
+	_, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	return err == nil
 }
 
 // MakeRaw put the terminal connected to the given file descriptor into raw
 // mode and returns the previous state of the terminal so that it can be
 // restored.
 func MakeRaw(fd int) (*State, error) {
-	var oldState State
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
+	termios, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	if err != nil {
 		return nil, err
 	}
 
-	newState := oldState.termios
+	oldState := State{termios: *termios}
+
 	// This attempts to replicate the behaviour documented for cfmakeraw in
 	// the termios(3) manpage.
-	newState.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON
-	newState.Oflag &^= syscall.OPOST
-	newState.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN
-	newState.Cflag &^= syscall.CSIZE | syscall.PARENB
-	newState.Cflag |= syscall.CS8
-	newState.Cc[unix.VMIN] = 1
-	newState.Cc[unix.VTIME] = 0
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
+	termios.Iflag &^= unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON
+	termios.Oflag &^= unix.OPOST
+	termios.Lflag &^= unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN
+	termios.Cflag &^= unix.CSIZE | unix.PARENB
+	termios.Cflag |= unix.CS8
+	termios.Cc[unix.VMIN] = 1
+	termios.Cc[unix.VTIME] = 0
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, termios); err != nil {
 		return nil, err
 	}
 
@@ -64,59 +61,55 @@ func MakeRaw(fd int) (*State, error) {
 // GetState returns the current state of a terminal which may be useful to
 // restore the terminal after a signal.
 func GetState(fd int) (*State, error) {
-	var oldState State
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
+	termios, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	if err != nil {
 		return nil, err
 	}
 
-	return &oldState, nil
+	return &State{termios: *termios}, nil
 }
 
 // Restore restores the terminal connected to the given file descriptor to a
 // previous state.
 func Restore(fd int, state *State) error {
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0); err != 0 {
-		return err
-	}
-	return nil
+	return unix.IoctlSetTermios(fd, ioctlWriteTermios, &state.termios)
 }
 
 // GetSize returns the dimensions of the given terminal.
 func GetSize(fd int) (width, height int, err error) {
-	var dimensions [4]uint16
-
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
+	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
+	if err != nil {
 		return -1, -1, err
 	}
-	return int(dimensions[1]), int(dimensions[0]), nil
+	return int(ws.Col), int(ws.Row), nil
 }
 
 // passwordReader is an io.Reader that reads from a specific file descriptor.
 type passwordReader int
 
 func (r passwordReader) Read(buf []byte) (int, error) {
-	return syscall.Read(int(r), buf)
+	return unix.Read(int(r), buf)
 }
 
 // ReadPassword reads a line of input from a terminal without local echo.  This
 // is commonly used for inputting passwords and other sensitive data. The slice
 // returned does not include the \n.
 func ReadPassword(fd int) ([]byte, error) {
-	var oldState syscall.Termios
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); err != 0 {
+	termios, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	if err != nil {
 		return nil, err
 	}
 
-	newState := oldState
-	newState.Lflag &^= syscall.ECHO
-	newState.Lflag |= syscall.ICANON | syscall.ISIG
-	newState.Iflag |= syscall.ICRNL
-	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
+	newState := *termios
+	newState.Lflag &^= unix.ECHO
+	newState.Lflag |= unix.ICANON | unix.ISIG
+	newState.Iflag |= unix.ICRNL
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, &newState); err != nil {
 		return nil, err
 	}
 
 	defer func() {
-		syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0)
+		unix.IoctlSetTermios(fd, ioctlWriteTermios, termios)
 	}()
 
 	return readPasswordLine(passwordReader(fd))

vendor/golang.org/x/crypto/ssh/test/banner_test.go

@@ -0,0 +1,33 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd
+
+package test
+
+import (
+	"testing"
+
+)
+
+func TestBannerCallbackAgainstOpenSSH(t *testing.T) {
+	server := newServer(t)
+	defer server.Shutdown()
+
+	clientConf := clientConfig()
+
+	var receivedBanner string
+	clientConf.BannerCallback = func(message string) error {
+		receivedBanner = message
+		return nil
+	}
+
+	conn := server.Dial(clientConf)
+	defer conn.Close()
+
+	expected := "Server Banner"
+	if receivedBanner != expected {
+		t.Fatalf("got %v; want %v", receivedBanner, expected)
+	}
+}

vendor/golang.org/x/crypto/ssh/test/test_unix_test.go

@@ -27,6 +27,7 @@ import (
 
 const sshd_config = `
 Protocol 2
+Banner {{.Dir}}/banner
 HostKey {{.Dir}}/id_rsa
 HostKey {{.Dir}}/id_dsa
 HostKey {{.Dir}}/id_ecdsa
@@ -256,6 +257,8 @@ func newServer(t *testing.T) *server {
 	}
 	f.Close()
 
+	writeFile(filepath.Join(dir, "banner"), []byte("Server Banner"))
+
 	for k, v := range testdata.PEMBytes {
 		filename := "id_" + k
 		writeFile(filepath.Join(dir, filename), v)

vendor/golang.org/x/crypto/ssh/transport.go

@@ -254,7 +254,7 @@ func newPacketCipher(d direction, algs directionAlgorithms, kex *kexResult) (pac
 	iv, key, macKey := generateKeys(d, algs, kex)
 
 	if algs.Cipher == gcmCipherID {
-		return newGCMCipher(iv, key, macKey)
+		return newGCMCipher(iv, key)
 	}
 
 	if algs.Cipher == aes128cbcID {