build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>

e2e/vendor/k8s.io/client-go/tools/remotecommand/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - aojea
+  - liggitt
+  - seans3
+reviewers:
+  - aojea
+  - liggitt
+  - seans3

e2e/vendor/k8s.io/client-go/tools/remotecommand/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package remotecommand adds support for executing commands in containers,
+// with support for separate stdin, stdout, and stderr streams, as well as
+// TTY.
+package remotecommand // import "k8s.io/client-go/tools/remotecommand"

e2e/vendor/k8s.io/client-go/tools/remotecommand/errorstream.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+
+	"k8s.io/apimachinery/pkg/util/runtime"
+)
+
+// errorStreamDecoder interprets the data on the error channel and creates a go error object from it.
+type errorStreamDecoder interface {
+	decode(message []byte) error
+}
+
+// watchErrorStream watches the errorStream for remote command error data,
+// decodes it with the given errorStreamDecoder, sends the decoded error (or nil if the remote
+// command exited successfully) to the returned error channel, and closes it.
+// This function returns immediately.
+func watchErrorStream(errorStream io.Reader, d errorStreamDecoder) chan error {
+	errorChan := make(chan error)
+
+	go func() {
+		defer runtime.HandleCrash()
+
+		message, err := io.ReadAll(errorStream)
+		switch {
+		case err != nil && err != io.EOF:
+			errorChan <- fmt.Errorf("error reading from error stream: %s", err)
+		case len(message) > 0:
+			errorChan <- d.decode(message)
+		default:
+			errorChan <- nil
+		}
+		close(errorChan)
+	}()
+
+	return errorChan
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/fallback.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"context"
+
+	"k8s.io/klog/v2"
+)
+
+var _ Executor = &FallbackExecutor{}
+
+type FallbackExecutor struct {
+	primary        Executor
+	secondary      Executor
+	shouldFallback func(error) bool
+}
+
+// NewFallbackExecutor creates an Executor that first attempts to use the
+// WebSocketExecutor, falling back to the legacy SPDYExecutor if the initial
+// websocket "StreamWithContext" call fails.
+// func NewFallbackExecutor(config *restclient.Config, method string, url *url.URL) (Executor, error) {
+func NewFallbackExecutor(primary, secondary Executor, shouldFallback func(error) bool) (Executor, error) {
+	return &FallbackExecutor{
+		primary:        primary,
+		secondary:      secondary,
+		shouldFallback: shouldFallback,
+	}, nil
+}
+
+// Stream is deprecated. Please use "StreamWithContext".
+func (f *FallbackExecutor) Stream(options StreamOptions) error {
+	return f.StreamWithContext(context.Background(), options)
+}
+
+// StreamWithContext initially attempts to call "StreamWithContext" using the
+// primary executor, falling back to calling the secondary executor if the
+// initial primary call to upgrade to a websocket connection fails.
+func (f *FallbackExecutor) StreamWithContext(ctx context.Context, options StreamOptions) error {
+	err := f.primary.StreamWithContext(ctx, options)
+	if f.shouldFallback(err) {
+		klog.V(4).Infof("RemoteCommand fallback: %v", err)
+		return f.secondary.StreamWithContext(ctx, options)
+	}
+	return err
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/reader.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"io"
+)
+
+// readerWrapper delegates to an io.Reader so that only the io.Reader interface is implemented,
+// to keep io.Copy from doing things we don't want when copying from the reader to the data stream.
+//
+// If the Stdin io.Reader provided to remotecommand implements a WriteTo function (like bytes.Buffer does[1]),
+// io.Copy calls that method[2] to attempt to write the entire buffer to the stream in one call.
+// That results in an oversized call to spdystream.Stream#Write [3],
+// which results in a single oversized data frame[4] that is too large.
+//
+// [1] https://golang.org/pkg/bytes/#Buffer.WriteTo
+// [2] https://golang.org/pkg/io/#Copy
+// [3] https://github.com/kubernetes/kubernetes/blob/90295640ef87db9daa0144c5617afe889e7992b2/vendor/github.com/docker/spdystream/stream.go#L66-L73
+// [4] https://github.com/kubernetes/kubernetes/blob/90295640ef87db9daa0144c5617afe889e7992b2/vendor/github.com/docker/spdystream/spdy/write.go#L302-L304
+type readerWrapper struct {
+	reader io.Reader
+}
+
+func (r readerWrapper) Read(p []byte) (int, error) {
+	return r.reader.Read(p)
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/remotecommand.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"k8s.io/apimachinery/pkg/util/httpstream"
+)
+
+// StreamOptions holds information pertaining to the current streaming session:
+// input/output streams, if the client is requesting a TTY, and a terminal size queue to
+// support terminal resizing.
+type StreamOptions struct {
+	Stdin             io.Reader
+	Stdout            io.Writer
+	Stderr            io.Writer
+	Tty               bool
+	TerminalSizeQueue TerminalSizeQueue
+}
+
+// Executor is an interface for transporting shell-style streams.
+type Executor interface {
+	// Deprecated: use StreamWithContext instead to avoid possible resource leaks.
+	// See https://github.com/kubernetes/kubernetes/pull/103177 for details.
+	Stream(options StreamOptions) error
+
+	// StreamWithContext initiates the transport of the standard shell streams. It will
+	// transport any non-nil stream to a remote system, and return an error if a problem
+	// occurs. If tty is set, the stderr stream is not used (raw TTY manages stdout and
+	// stderr over the stdout stream).
+	// The context controls the entire lifetime of stream execution.
+	StreamWithContext(ctx context.Context, options StreamOptions) error
+}
+
+type streamCreator interface {
+	CreateStream(headers http.Header) (httpstream.Stream, error)
+}
+
+type streamProtocolHandler interface {
+	stream(conn streamCreator) error
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/resize.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+// TerminalSize and TerminalSizeQueue was a part of k8s.io/kubernetes/pkg/util/term
+// and were moved in order to decouple client from other term dependencies
+
+// TerminalSize represents the width and height of a terminal.
+type TerminalSize struct {
+	Width  uint16
+	Height uint16
+}
+
+// TerminalSizeQueue is capable of returning terminal resize events as they occur.
+type TerminalSizeQueue interface {
+	// Next returns the new terminal size after the terminal has been resized. It returns nil when
+	// monitoring has been stopped.
+	Next() *TerminalSize
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/spdy.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/apimachinery/pkg/util/remotecommand"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/transport/spdy"
+	"k8s.io/klog/v2"
+)
+
+// spdyStreamExecutor handles transporting standard shell streams over an httpstream connection.
+type spdyStreamExecutor struct {
+	upgrader  spdy.Upgrader
+	transport http.RoundTripper
+
+	method          string
+	url             *url.URL
+	protocols       []string
+	rejectRedirects bool // if true, receiving redirect from upstream is an error
+}
+
+// NewSPDYExecutor connects to the provided server and upgrades the connection to
+// multiplexed bidirectional streams.
+func NewSPDYExecutor(config *restclient.Config, method string, url *url.URL) (Executor, error) {
+	wrapper, upgradeRoundTripper, err := spdy.RoundTripperFor(config)
+	if err != nil {
+		return nil, err
+	}
+	return NewSPDYExecutorForTransports(wrapper, upgradeRoundTripper, method, url)
+}
+
+// NewSPDYExecutorRejectRedirects returns an Executor that will upgrade the future
+// connection to a SPDY bi-directional streaming connection when calling "Stream" (deprecated)
+// or "StreamWithContext" (preferred). Additionally, if the upstream server returns a redirect
+// during the attempted upgrade in these "Stream" calls, an error is returned.
+func NewSPDYExecutorRejectRedirects(transport http.RoundTripper, upgrader spdy.Upgrader, method string, url *url.URL) (Executor, error) {
+	executor, err := NewSPDYExecutorForTransports(transport, upgrader, method, url)
+	if err != nil {
+		return nil, err
+	}
+	spdyExecutor := executor.(*spdyStreamExecutor)
+	spdyExecutor.rejectRedirects = true
+	return spdyExecutor, nil
+}
+
+// NewSPDYExecutorForTransports connects to the provided server using the given transport,
+// upgrades the response using the given upgrader to multiplexed bidirectional streams.
+func NewSPDYExecutorForTransports(transport http.RoundTripper, upgrader spdy.Upgrader, method string, url *url.URL) (Executor, error) {
+	return NewSPDYExecutorForProtocols(
+		transport, upgrader, method, url,
+		remotecommand.StreamProtocolV5Name,
+		remotecommand.StreamProtocolV4Name,
+		remotecommand.StreamProtocolV3Name,
+		remotecommand.StreamProtocolV2Name,
+		remotecommand.StreamProtocolV1Name,
+	)
+}
+
+// NewSPDYExecutorForProtocols connects to the provided server and upgrades the connection to
+// multiplexed bidirectional streams using only the provided protocols. Exposed for testing, most
+// callers should use NewSPDYExecutor or NewSPDYExecutorForTransports.
+func NewSPDYExecutorForProtocols(transport http.RoundTripper, upgrader spdy.Upgrader, method string, url *url.URL, protocols ...string) (Executor, error) {
+	return &spdyStreamExecutor{
+		upgrader:  upgrader,
+		transport: transport,
+		method:    method,
+		url:       url,
+		protocols: protocols,
+	}, nil
+}
+
+// Stream opens a protocol streamer to the server and streams until a client closes
+// the connection or the server disconnects.
+func (e *spdyStreamExecutor) Stream(options StreamOptions) error {
+	return e.StreamWithContext(context.Background(), options)
+}
+
+// newConnectionAndStream creates a new SPDY connection and a stream protocol handler upon it.
+func (e *spdyStreamExecutor) newConnectionAndStream(ctx context.Context, options StreamOptions) (httpstream.Connection, streamProtocolHandler, error) {
+	req, err := http.NewRequestWithContext(ctx, e.method, e.url.String(), nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating request: %v", err)
+	}
+
+	client := http.Client{Transport: e.transport}
+	if e.rejectRedirects {
+		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return fmt.Errorf("redirect not allowed")
+		}
+	}
+	conn, protocol, err := spdy.Negotiate(
+		e.upgrader,
+		&client,
+		req,
+		e.protocols...,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var streamer streamProtocolHandler
+
+	switch protocol {
+	case remotecommand.StreamProtocolV5Name:
+		streamer = newStreamProtocolV5(options)
+	case remotecommand.StreamProtocolV4Name:
+		streamer = newStreamProtocolV4(options)
+	case remotecommand.StreamProtocolV3Name:
+		streamer = newStreamProtocolV3(options)
+	case remotecommand.StreamProtocolV2Name:
+		streamer = newStreamProtocolV2(options)
+	case "":
+		klog.V(4).Infof("The server did not negotiate a streaming protocol version. Falling back to %s", remotecommand.StreamProtocolV1Name)
+		fallthrough
+	case remotecommand.StreamProtocolV1Name:
+		streamer = newStreamProtocolV1(options)
+	}
+
+	return conn, streamer, nil
+}
+
+// StreamWithContext opens a protocol streamer to the server and streams until a client closes
+// the connection or the server disconnects or the context is done.
+func (e *spdyStreamExecutor) StreamWithContext(ctx context.Context, options StreamOptions) error {
+	conn, streamer, err := e.newConnectionAndStream(ctx, options)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	panicChan := make(chan any, 1)
+	errorChan := make(chan error, 1)
+	go func() {
+		defer func() {
+			if p := recover(); p != nil {
+				panicChan <- p
+			}
+		}()
+		errorChan <- streamer.stream(conn)
+	}()
+
+	select {
+	case p := <-panicChan:
+		panic(p)
+	case err := <-errorChan:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/v1.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/klog/v2"
+)
+
+// streamProtocolV1 implements the first version of the streaming exec & attach
+// protocol. This version has some bugs, such as not being able to detect when
+// non-interactive stdin data has ended. See https://issues.k8s.io/13394 and
+// https://issues.k8s.io/13395 for more details.
+type streamProtocolV1 struct {
+	StreamOptions
+
+	errorStream  httpstream.Stream
+	remoteStdin  httpstream.Stream
+	remoteStdout httpstream.Stream
+	remoteStderr httpstream.Stream
+}
+
+var _ streamProtocolHandler = &streamProtocolV1{}
+
+func newStreamProtocolV1(options StreamOptions) streamProtocolHandler {
+	return &streamProtocolV1{
+		StreamOptions: options,
+	}
+}
+
+func (p *streamProtocolV1) stream(conn streamCreator) error {
+	doneChan := make(chan struct{}, 2)
+	errorChan := make(chan error)
+
+	cp := func(s string, dst io.Writer, src io.Reader) {
+		klog.V(6).Infof("Copying %s", s)
+		defer klog.V(6).Infof("Done copying %s", s)
+		if _, err := io.Copy(dst, src); err != nil && err != io.EOF {
+			klog.Errorf("Error copying %s: %v", s, err)
+		}
+		if s == v1.StreamTypeStdout || s == v1.StreamTypeStderr {
+			doneChan <- struct{}{}
+		}
+	}
+
+	// set up all the streams first
+	var err error
+	headers := http.Header{}
+	headers.Set(v1.StreamType, v1.StreamTypeError)
+	p.errorStream, err = conn.CreateStream(headers)
+	if err != nil {
+		return err
+	}
+	defer p.errorStream.Reset()
+
+	// Create all the streams first, then start the copy goroutines. The server doesn't start its copy
+	// goroutines until it's received all of the streams. If the client creates the stdin stream and
+	// immediately begins copying stdin data to the server, it's possible to overwhelm and wedge the
+	// spdy frame handler in the server so that it is full of unprocessed frames. The frames aren't
+	// getting processed because the server hasn't started its copying, and it won't do that until it
+	// gets all the streams. By creating all the streams first, we ensure that the server is ready to
+	// process data before the client starts sending any. See https://issues.k8s.io/16373 for more info.
+	if p.Stdin != nil {
+		headers.Set(v1.StreamType, v1.StreamTypeStdin)
+		p.remoteStdin, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+		defer p.remoteStdin.Reset()
+	}
+
+	if p.Stdout != nil {
+		headers.Set(v1.StreamType, v1.StreamTypeStdout)
+		p.remoteStdout, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+		defer p.remoteStdout.Reset()
+	}
+
+	if p.Stderr != nil && !p.Tty {
+		headers.Set(v1.StreamType, v1.StreamTypeStderr)
+		p.remoteStderr, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+		defer p.remoteStderr.Reset()
+	}
+
+	// now that all the streams have been created, proceed with reading & copying
+
+	// always read from errorStream
+	go func() {
+		message, err := io.ReadAll(p.errorStream)
+		if err != nil && err != io.EOF {
+			errorChan <- fmt.Errorf("Error reading from error stream: %s", err)
+			return
+		}
+		if len(message) > 0 {
+			errorChan <- fmt.Errorf("Error executing remote command: %s", message)
+			return
+		}
+	}()
+
+	if p.Stdin != nil {
+		// TODO this goroutine will never exit cleanly (the io.Copy never unblocks)
+		// because stdin is not closed until the process exits. If we try to call
+		// stdin.Close(), it returns no error but doesn't unblock the copy. It will
+		// exit when the process exits, instead.
+		go cp(v1.StreamTypeStdin, p.remoteStdin, readerWrapper{p.Stdin})
+	}
+
+	waitCount := 0
+	completedStreams := 0
+
+	if p.Stdout != nil {
+		waitCount++
+		go cp(v1.StreamTypeStdout, p.Stdout, p.remoteStdout)
+	}
+
+	if p.Stderr != nil && !p.Tty {
+		waitCount++
+		go cp(v1.StreamTypeStderr, p.Stderr, p.remoteStderr)
+	}
+
+Loop:
+	for {
+		select {
+		case <-doneChan:
+			completedStreams++
+			if completedStreams == waitCount {
+				break Loop
+			}
+		case err := <-errorChan:
+			return err
+		}
+	}
+
+	return nil
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/v2.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/runtime"
+)
+
+// streamProtocolV2 implements version 2 of the streaming protocol for attach
+// and exec. The original streaming protocol was metav1. As a result, this
+// version is referred to as version 2, even though it is the first actual
+// numbered version.
+type streamProtocolV2 struct {
+	StreamOptions
+
+	errorStream  io.Reader
+	remoteStdin  io.ReadWriteCloser
+	remoteStdout io.Reader
+	remoteStderr io.Reader
+}
+
+var _ streamProtocolHandler = &streamProtocolV2{}
+
+func newStreamProtocolV2(options StreamOptions) streamProtocolHandler {
+	return &streamProtocolV2{
+		StreamOptions: options,
+	}
+}
+
+func (p *streamProtocolV2) createStreams(conn streamCreator) error {
+	var err error
+	headers := http.Header{}
+
+	// set up error stream
+	headers.Set(v1.StreamType, v1.StreamTypeError)
+	p.errorStream, err = conn.CreateStream(headers)
+	if err != nil {
+		return err
+	}
+
+	// set up stdin stream
+	if p.Stdin != nil {
+		headers.Set(v1.StreamType, v1.StreamTypeStdin)
+		p.remoteStdin, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	// set up stdout stream
+	if p.Stdout != nil {
+		headers.Set(v1.StreamType, v1.StreamTypeStdout)
+		p.remoteStdout, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	// set up stderr stream
+	if p.Stderr != nil && !p.Tty {
+		headers.Set(v1.StreamType, v1.StreamTypeStderr)
+		p.remoteStderr, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *streamProtocolV2) copyStdin() {
+	if p.Stdin != nil {
+		var once sync.Once
+
+		// copy from client's stdin to container's stdin
+		go func() {
+			defer runtime.HandleCrash()
+
+			// if p.stdin is noninteractive, p.g. `echo abc | kubectl exec -i <pod> -- cat`, make sure
+			// we close remoteStdin as soon as the copy from p.stdin to remoteStdin finishes. Otherwise
+			// the executed command will remain running.
+			defer once.Do(func() { p.remoteStdin.Close() })
+
+			if _, err := io.Copy(p.remoteStdin, readerWrapper{p.Stdin}); err != nil {
+				runtime.HandleError(err)
+			}
+		}()
+
+		// read from remoteStdin until the stream is closed. this is essential to
+		// be able to exit interactive sessions cleanly and not leak goroutines or
+		// hang the client's terminal.
+		//
+		// TODO we aren't using go-dockerclient any more; revisit this to determine if it's still
+		// required by engine-api.
+		//
+		// go-dockerclient's current hijack implementation
+		// (https://github.com/fsouza/go-dockerclient/blob/89f3d56d93788dfe85f864a44f85d9738fca0670/client.go#L564)
+		// waits for all three streams (stdin/stdout/stderr) to finish copying
+		// before returning. When hijack finishes copying stdout/stderr, it calls
+		// Close() on its side of remoteStdin, which allows this copy to complete.
+		// When that happens, we must Close() on our side of remoteStdin, to
+		// allow the copy in hijack to complete, and hijack to return.
+		go func() {
+			defer runtime.HandleCrash()
+			defer once.Do(func() { p.remoteStdin.Close() })
+
+			// this "copy" doesn't actually read anything - it's just here to wait for
+			// the server to close remoteStdin.
+			if _, err := io.Copy(io.Discard, p.remoteStdin); err != nil {
+				runtime.HandleError(err)
+			}
+		}()
+	}
+}
+
+func (p *streamProtocolV2) copyStdout(wg *sync.WaitGroup) {
+	if p.Stdout == nil {
+		return
+	}
+
+	wg.Add(1)
+	go func() {
+		defer runtime.HandleCrash()
+		defer wg.Done()
+		// make sure, packet in queue can be consumed.
+		// block in queue may lead to deadlock in conn.server
+		// issue: https://github.com/kubernetes/kubernetes/issues/96339
+		defer io.Copy(io.Discard, p.remoteStdout)
+
+		if _, err := io.Copy(p.Stdout, p.remoteStdout); err != nil {
+			runtime.HandleError(err)
+		}
+	}()
+}
+
+func (p *streamProtocolV2) copyStderr(wg *sync.WaitGroup) {
+	if p.Stderr == nil || p.Tty {
+		return
+	}
+
+	wg.Add(1)
+	go func() {
+		defer runtime.HandleCrash()
+		defer wg.Done()
+		defer io.Copy(io.Discard, p.remoteStderr)
+
+		if _, err := io.Copy(p.Stderr, p.remoteStderr); err != nil {
+			runtime.HandleError(err)
+		}
+	}()
+}
+
+func (p *streamProtocolV2) stream(conn streamCreator) error {
+	if err := p.createStreams(conn); err != nil {
+		return err
+	}
+
+	// now that all the streams have been created, proceed with reading & copying
+
+	errorChan := watchErrorStream(p.errorStream, &errorDecoderV2{})
+
+	p.copyStdin()
+
+	var wg sync.WaitGroup
+	p.copyStdout(&wg)
+	p.copyStderr(&wg)
+
+	// we're waiting for stdout/stderr to finish copying
+	wg.Wait()
+
+	// waits for errorStream to finish reading with an error or nil
+	return <-errorChan
+}
+
+// errorDecoderV2 interprets the error channel data as plain text.
+type errorDecoderV2 struct{}
+
+func (d *errorDecoderV2) decode(message []byte) error {
+	return fmt.Errorf("error executing remote command: %s", message)
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/v3.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/runtime"
+)
+
+// streamProtocolV3 implements version 3 of the streaming protocol for attach
+// and exec. This version adds support for resizing the container's terminal.
+type streamProtocolV3 struct {
+	*streamProtocolV2
+
+	resizeStream io.Writer
+}
+
+var _ streamProtocolHandler = &streamProtocolV3{}
+
+func newStreamProtocolV3(options StreamOptions) streamProtocolHandler {
+	return &streamProtocolV3{
+		streamProtocolV2: newStreamProtocolV2(options).(*streamProtocolV2),
+	}
+}
+
+func (p *streamProtocolV3) createStreams(conn streamCreator) error {
+	// set up the streams from v2
+	if err := p.streamProtocolV2.createStreams(conn); err != nil {
+		return err
+	}
+
+	// set up resize stream
+	if p.Tty {
+		headers := http.Header{}
+		headers.Set(v1.StreamType, v1.StreamTypeResize)
+		var err error
+		p.resizeStream, err = conn.CreateStream(headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *streamProtocolV3) handleResizes() {
+	if p.resizeStream == nil || p.TerminalSizeQueue == nil {
+		return
+	}
+	go func() {
+		defer runtime.HandleCrash()
+
+		encoder := json.NewEncoder(p.resizeStream)
+		for {
+			size := p.TerminalSizeQueue.Next()
+			if size == nil {
+				return
+			}
+			if err := encoder.Encode(&size); err != nil {
+				runtime.HandleError(err)
+			}
+		}
+	}()
+}
+
+func (p *streamProtocolV3) stream(conn streamCreator) error {
+	if err := p.createStreams(conn); err != nil {
+		return err
+	}
+
+	// now that all the streams have been created, proceed with reading & copying
+
+	errorChan := watchErrorStream(p.errorStream, &errorDecoderV3{})
+
+	p.handleResizes()
+
+	p.copyStdin()
+
+	var wg sync.WaitGroup
+	p.copyStdout(&wg)
+	p.copyStderr(&wg)
+
+	// we're waiting for stdout/stderr to finish copying
+	wg.Wait()
+
+	// waits for errorStream to finish reading with an error or nil
+	return <-errorChan
+}
+
+type errorDecoderV3 struct {
+	errorDecoderV2
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/v4.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"sync"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/client-go/util/exec"
+)
+
+// streamProtocolV4 implements version 4 of the streaming protocol for attach
+// and exec. This version adds support for exit codes on the error stream through
+// the use of metav1.Status instead of plain text messages.
+type streamProtocolV4 struct {
+	*streamProtocolV3
+}
+
+var _ streamProtocolHandler = &streamProtocolV4{}
+
+func newStreamProtocolV4(options StreamOptions) streamProtocolHandler {
+	return &streamProtocolV4{
+		streamProtocolV3: newStreamProtocolV3(options).(*streamProtocolV3),
+	}
+}
+
+func (p *streamProtocolV4) createStreams(conn streamCreator) error {
+	return p.streamProtocolV3.createStreams(conn)
+}
+
+func (p *streamProtocolV4) handleResizes() {
+	p.streamProtocolV3.handleResizes()
+}
+
+func (p *streamProtocolV4) stream(conn streamCreator) error {
+	if err := p.createStreams(conn); err != nil {
+		return err
+	}
+
+	// now that all the streams have been created, proceed with reading & copying
+
+	errorChan := watchErrorStream(p.errorStream, &errorDecoderV4{})
+
+	p.handleResizes()
+
+	p.copyStdin()
+
+	var wg sync.WaitGroup
+	p.copyStdout(&wg)
+	p.copyStderr(&wg)
+
+	// we're waiting for stdout/stderr to finish copying
+	wg.Wait()
+
+	// waits for errorStream to finish reading with an error or nil
+	return <-errorChan
+}
+
+// errorDecoderV4 interprets the json-marshaled metav1.Status on the error channel
+// and creates an exec.ExitError from it.
+type errorDecoderV4 struct{}
+
+func (d *errorDecoderV4) decode(message []byte) error {
+	status := metav1.Status{}
+	err := json.Unmarshal(message, &status)
+	if err != nil {
+		return fmt.Errorf("error stream protocol error: %v in %q", err, string(message))
+	}
+	switch status.Status {
+	case metav1.StatusSuccess:
+		return nil
+	case metav1.StatusFailure:
+		if status.Reason == remotecommand.NonZeroExitCodeReason {
+			if status.Details == nil {
+				return errors.New("error stream protocol error: details must be set")
+			}
+			for i := range status.Details.Causes {
+				c := &status.Details.Causes[i]
+				if c.Type != remotecommand.ExitCodeCauseType {
+					continue
+				}
+
+				rc, err := strconv.ParseUint(c.Message, 10, 8)
+				if err != nil {
+					return fmt.Errorf("error stream protocol error: invalid exit code value %q", c.Message)
+				}
+				return exec.CodeExitError{
+					Err:  fmt.Errorf("command terminated with exit code %d", rc),
+					Code: int(rc),
+				}
+			}
+
+			return fmt.Errorf("error stream protocol error: no %s cause given", remotecommand.ExitCodeCauseType)
+		}
+	default:
+		return errors.New("error stream protocol error: unknown error")
+	}
+
+	return errors.New(status.Message)
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/v5.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+// streamProtocolV5 add support for V5 of the remote command subprotocol.
+// For the streamProtocolHandler, this version is the same as V4.
+type streamProtocolV5 struct {
+	*streamProtocolV4
+}
+
+var _ streamProtocolHandler = &streamProtocolV5{}
+
+func newStreamProtocolV5(options StreamOptions) streamProtocolHandler {
+	return &streamProtocolV5{
+		streamProtocolV4: newStreamProtocolV4(options).(*streamProtocolV4),
+	}
+}
+
+func (p *streamProtocolV5) stream(conn streamCreator) error {
+	return p.streamProtocolV4.stream(conn)
+}

e2e/vendor/k8s.io/client-go/tools/remotecommand/websocket.go

@@ -0,0 +1,515 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	gwebsocket "github.com/gorilla/websocket"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/apimachinery/pkg/util/remotecommand"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/transport/websocket"
+	"k8s.io/klog/v2"
+)
+
+// writeDeadline defines the time that a client-side write to the websocket
+// connection must complete before an i/o timeout occurs.
+const writeDeadline = 60 * time.Second
+
+var (
+	_ Executor          = &wsStreamExecutor{}
+	_ streamCreator     = &wsStreamCreator{}
+	_ httpstream.Stream = &stream{}
+
+	streamType2streamID = map[string]byte{
+		v1.StreamTypeStdin:  remotecommand.StreamStdIn,
+		v1.StreamTypeStdout: remotecommand.StreamStdOut,
+		v1.StreamTypeStderr: remotecommand.StreamStdErr,
+		v1.StreamTypeError:  remotecommand.StreamErr,
+		v1.StreamTypeResize: remotecommand.StreamResize,
+	}
+)
+
+const (
+	// pingPeriod defines how often a heartbeat "ping" message is sent.
+	pingPeriod = 5 * time.Second
+	// pingReadDeadline defines the time waiting for a response heartbeat
+	// "pong" message before a timeout error occurs for websocket reading.
+	// This duration must always be greater than the "pingPeriod". By defining
+	// this deadline in terms of the ping period, we are essentially saying
+	// we can drop "X" (e.g. 12) pings before firing the timeout.
+	pingReadDeadline = (pingPeriod * 12) + (1 * time.Second)
+)
+
+// wsStreamExecutor handles transporting standard shell streams over an httpstream connection.
+type wsStreamExecutor struct {
+	transport http.RoundTripper
+	upgrader  websocket.ConnectionHolder
+	method    string
+	url       string
+	// requested protocols in priority order (e.g. v5.channel.k8s.io before v4.channel.k8s.io).
+	protocols []string
+	// selected protocol from the handshake process; could be empty string if handshake fails.
+	negotiated string
+	// period defines how often a "ping" heartbeat message is sent to the other endpoint.
+	heartbeatPeriod time.Duration
+	// deadline defines the amount of time before "pong" response must be received.
+	heartbeatDeadline time.Duration
+}
+
+func NewWebSocketExecutor(config *restclient.Config, method, url string) (Executor, error) {
+	// Only supports V5 protocol for correct version skew functionality.
+	// Previous api servers will proxy upgrade requests to legacy websocket
+	// servers on container runtimes which support V1-V4. These legacy
+	// websocket servers will not handle the new CLOSE signal.
+	return NewWebSocketExecutorForProtocols(config, method, url, remotecommand.StreamProtocolV5Name)
+}
+
+// NewWebSocketExecutorForProtocols allows to execute commands via a WebSocket connection.
+func NewWebSocketExecutorForProtocols(config *restclient.Config, method, url string, protocols ...string) (Executor, error) {
+	transport, upgrader, err := websocket.RoundTripperFor(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating websocket transports: %v", err)
+	}
+	return &wsStreamExecutor{
+		transport:         transport,
+		upgrader:          upgrader,
+		method:            method,
+		url:               url,
+		protocols:         protocols,
+		heartbeatPeriod:   pingPeriod,
+		heartbeatDeadline: pingReadDeadline,
+	}, nil
+}
+
+// Deprecated: use StreamWithContext instead to avoid possible resource leaks.
+// See https://github.com/kubernetes/kubernetes/pull/103177 for details.
+func (e *wsStreamExecutor) Stream(options StreamOptions) error {
+	return e.StreamWithContext(context.Background(), options)
+}
+
+// StreamWithContext upgrades an HTTPRequest to a WebSocket connection, and starts the various
+// goroutines to implement the necessary streams over the connection. The "options" parameter
+// defines which streams are requested. Returns an error if one occurred. This method is NOT
+// safe to run concurrently with the same executor (because of the state stored in the upgrader).
+func (e *wsStreamExecutor) StreamWithContext(ctx context.Context, options StreamOptions) error {
+	req, err := http.NewRequestWithContext(ctx, e.method, e.url, nil)
+	if err != nil {
+		return err
+	}
+	conn, err := websocket.Negotiate(e.transport, e.upgrader, req, e.protocols...)
+	if err != nil {
+		return err
+	}
+	if conn == nil {
+		panic(fmt.Errorf("websocket connection is nil"))
+	}
+	defer conn.Close()
+	e.negotiated = conn.Subprotocol()
+	klog.V(4).Infof("The subprotocol is %s", e.negotiated)
+
+	var streamer streamProtocolHandler
+	switch e.negotiated {
+	case remotecommand.StreamProtocolV5Name:
+		streamer = newStreamProtocolV5(options)
+	case remotecommand.StreamProtocolV4Name:
+		streamer = newStreamProtocolV4(options)
+	case remotecommand.StreamProtocolV3Name:
+		streamer = newStreamProtocolV3(options)
+	case remotecommand.StreamProtocolV2Name:
+		streamer = newStreamProtocolV2(options)
+	case "":
+		klog.V(4).Infof("The server did not negotiate a streaming protocol version. Falling back to %s", remotecommand.StreamProtocolV1Name)
+		fallthrough
+	case remotecommand.StreamProtocolV1Name:
+		streamer = newStreamProtocolV1(options)
+	}
+
+	panicChan := make(chan any, 1)
+	errorChan := make(chan error, 1)
+	go func() {
+		defer func() {
+			if p := recover(); p != nil {
+				panicChan <- p
+			}
+		}()
+		creator := newWSStreamCreator(conn)
+		go creator.readDemuxLoop(
+			e.upgrader.DataBufferSize(),
+			e.heartbeatPeriod,
+			e.heartbeatDeadline,
+		)
+		errorChan <- streamer.stream(creator)
+	}()
+
+	select {
+	case p := <-panicChan:
+		panic(p)
+	case err := <-errorChan:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+type wsStreamCreator struct {
+	conn *gwebsocket.Conn
+	// Protects writing to websocket connection; reading is lock-free
+	connWriteLock sync.Mutex
+	// map of stream id to stream; multiple streams read/write the connection
+	streams   map[byte]*stream
+	streamsMu sync.Mutex
+	// setStreamErr holds the error to return to anyone calling setStreams.
+	// this is populated in closeAllStreamReaders
+	setStreamErr error
+}
+
+func newWSStreamCreator(conn *gwebsocket.Conn) *wsStreamCreator {
+	return &wsStreamCreator{
+		conn:    conn,
+		streams: map[byte]*stream{},
+	}
+}
+
+func (c *wsStreamCreator) getStream(id byte) *stream {
+	c.streamsMu.Lock()
+	defer c.streamsMu.Unlock()
+	return c.streams[id]
+}
+
+func (c *wsStreamCreator) setStream(id byte, s *stream) error {
+	c.streamsMu.Lock()
+	defer c.streamsMu.Unlock()
+	if c.setStreamErr != nil {
+		return c.setStreamErr
+	}
+	c.streams[id] = s
+	return nil
+}
+
+// CreateStream uses id from passed headers to create a stream over "c.conn" connection.
+// Returns a Stream structure or nil and an error if one occurred.
+func (c *wsStreamCreator) CreateStream(headers http.Header) (httpstream.Stream, error) {
+	streamType := headers.Get(v1.StreamType)
+	id, ok := streamType2streamID[streamType]
+	if !ok {
+		return nil, fmt.Errorf("unknown stream type: %s", streamType)
+	}
+	if s := c.getStream(id); s != nil {
+		return nil, fmt.Errorf("duplicate stream for type %s", streamType)
+	}
+	reader, writer := io.Pipe()
+	s := &stream{
+		headers:       headers,
+		readPipe:      reader,
+		writePipe:     writer,
+		conn:          c.conn,
+		connWriteLock: &c.connWriteLock,
+		id:            id,
+	}
+	if err := c.setStream(id, s); err != nil {
+		_ = s.writePipe.Close()
+		_ = s.readPipe.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+// readDemuxLoop is the lock-free reading processor for this endpoint of the websocket
+// connection. This loop reads the connection, and demultiplexes the data
+// into one of the individual stream pipes (by checking the stream id). This
+// loop can *not* be run concurrently, because there can only be one websocket
+// connection reader at a time (a read mutex would provide no benefit).
+func (c *wsStreamCreator) readDemuxLoop(bufferSize int, period time.Duration, deadline time.Duration) {
+	// Initialize and start the ping/pong heartbeat.
+	h := newHeartbeat(c.conn, period, deadline)
+	// Set initial timeout for websocket connection reading.
+	if err := c.conn.SetReadDeadline(time.Now().Add(deadline)); err != nil {
+		klog.Errorf("Websocket initial setting read deadline failed %v", err)
+		return
+	}
+	go h.start()
+	// Buffer size must correspond to the same size allocated
+	// for the read buffer during websocket client creation. A
+	// difference can cause incomplete connection reads.
+	readBuffer := make([]byte, bufferSize)
+	for {
+		// NextReader() only returns data messages (BinaryMessage or Text
+		// Message). Even though this call will never return control frames
+		// such as ping, pong, or close, this call is necessary for these
+		// message types to be processed. There can only be one reader
+		// at a time, so this reader loop must *not* be run concurrently;
+		// there is no lock for reading. Calling "NextReader()" before the
+		// current reader has been processed will close the current reader.
+		// If the heartbeat read deadline times out, this "NextReader()" will
+		// return an i/o error, and error handling will clean up.
+		messageType, r, err := c.conn.NextReader()
+		if err != nil {
+			websocketErr, ok := err.(*gwebsocket.CloseError)
+			if ok && websocketErr.Code == gwebsocket.CloseNormalClosure {
+				err = nil // readers will get io.EOF as it's a normal closure
+			} else {
+				err = fmt.Errorf("next reader: %w", err)
+			}
+			c.closeAllStreamReaders(err)
+			return
+		}
+		// All remote command protocols send/receive only binary data messages.
+		if messageType != gwebsocket.BinaryMessage {
+			c.closeAllStreamReaders(fmt.Errorf("unexpected message type: %d", messageType))
+			return
+		}
+		// It's ok to read just a single byte because the underlying library wraps the actual
+		// connection with a buffered reader anyway.
+		_, err = io.ReadFull(r, readBuffer[:1])
+		if err != nil {
+			c.closeAllStreamReaders(fmt.Errorf("read stream id: %w", err))
+			return
+		}
+		streamID := readBuffer[0]
+		s := c.getStream(streamID)
+		if s == nil {
+			klog.Errorf("Unknown stream id %d, discarding message", streamID)
+			continue
+		}
+		for {
+			nr, errRead := r.Read(readBuffer)
+			if nr > 0 {
+				// Write the data to the stream's pipe. This can block.
+				_, errWrite := s.writePipe.Write(readBuffer[:nr])
+				if errWrite != nil {
+					// Pipe must have been closed by the stream user.
+					// Nothing to do, discard the message.
+					break
+				}
+			}
+			if errRead != nil {
+				if errRead == io.EOF {
+					break
+				}
+				c.closeAllStreamReaders(fmt.Errorf("read message: %w", err))
+				return
+			}
+		}
+	}
+}
+
+// closeAllStreamReaders closes readers in all streams.
+// This unblocks all stream.Read() calls, and keeps any future streams from being created.
+func (c *wsStreamCreator) closeAllStreamReaders(err error) {
+	c.streamsMu.Lock()
+	defer c.streamsMu.Unlock()
+	for _, s := range c.streams {
+		// Closing writePipe unblocks all readPipe.Read() callers and prevents any future writes.
+		_ = s.writePipe.CloseWithError(err)
+	}
+	// ensure callers to setStreams receive an error after this point
+	if err != nil {
+		c.setStreamErr = err
+	} else {
+		c.setStreamErr = fmt.Errorf("closed all streams")
+	}
+}
+
+type stream struct {
+	headers   http.Header
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+	// conn is used for writing directly into the connection.
+	// Is nil after Close() / Reset() to prevent future writes.
+	conn *gwebsocket.Conn
+	// connWriteLock protects conn against concurrent write operations. There must be a single writer and a single reader only.
+	// The mutex is shared across all streams because the underlying connection is shared.
+	connWriteLock *sync.Mutex
+	id            byte
+}
+
+func (s *stream) Read(p []byte) (n int, err error) {
+	return s.readPipe.Read(p)
+}
+
+// Write writes directly to the underlying WebSocket connection.
+func (s *stream) Write(p []byte) (n int, err error) {
+	klog.V(4).Infof("Write() on stream %d", s.id)
+	defer klog.V(4).Infof("Write() done on stream %d", s.id)
+	s.connWriteLock.Lock()
+	defer s.connWriteLock.Unlock()
+	if s.conn == nil {
+		return 0, fmt.Errorf("write on closed stream %d", s.id)
+	}
+	err = s.conn.SetWriteDeadline(time.Now().Add(writeDeadline))
+	if err != nil {
+		klog.V(7).Infof("Websocket setting write deadline failed %v", err)
+		return 0, err
+	}
+	// Message writer buffers the message data, so we don't need to do that ourselves.
+	// Just write id and the data as two separate writes to avoid allocating an intermediate buffer.
+	w, err := s.conn.NextWriter(gwebsocket.BinaryMessage)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		if w != nil {
+			w.Close()
+		}
+	}()
+	_, err = w.Write([]byte{s.id})
+	if err != nil {
+		return 0, err
+	}
+	n, err = w.Write(p)
+	if err != nil {
+		return n, err
+	}
+	err = w.Close()
+	w = nil
+	return n, err
+}
+
+// Close half-closes the stream, indicating this side is finished with the stream.
+func (s *stream) Close() error {
+	klog.V(4).Infof("Close() on stream %d", s.id)
+	defer klog.V(4).Infof("Close() done on stream %d", s.id)
+	s.connWriteLock.Lock()
+	defer s.connWriteLock.Unlock()
+	if s.conn == nil {
+		return fmt.Errorf("Close() on already closed stream %d", s.id)
+	}
+	// Communicate the CLOSE stream signal to the other websocket endpoint.
+	err := s.conn.WriteMessage(gwebsocket.BinaryMessage, []byte{remotecommand.StreamClose, s.id})
+	s.conn = nil
+	return err
+}
+
+func (s *stream) Reset() error {
+	klog.V(4).Infof("Reset() on stream %d", s.id)
+	defer klog.V(4).Infof("Reset() done on stream %d", s.id)
+	s.Close()
+	return s.writePipe.Close()
+}
+
+func (s *stream) Headers() http.Header {
+	return s.headers
+}
+
+func (s *stream) Identifier() uint32 {
+	return uint32(s.id)
+}
+
+// heartbeat encasulates data necessary for the websocket ping/pong heartbeat. This
+// heartbeat works by setting a read deadline on the websocket connection, then
+// pushing this deadline into the future for every successful heartbeat. If the
+// heartbeat "pong" fails to respond within the deadline, then the "NextReader()" call
+// inside the "readDemuxLoop" will return an i/o error prompting a connection close
+// and cleanup.
+type heartbeat struct {
+	conn *gwebsocket.Conn
+	// period defines how often a "ping" heartbeat message is sent to the other endpoint
+	period time.Duration
+	// closing the "closer" channel will clean up the heartbeat timers
+	closer chan struct{}
+	// optional data to send with "ping" message
+	message []byte
+	// optionally received data message with "pong" message, same as sent with ping
+	pongMessage []byte
+}
+
+// newHeartbeat creates heartbeat structure encapsulating fields necessary to
+// run the websocket connection ping/pong mechanism and sets up handlers on
+// the websocket connection.
+func newHeartbeat(conn *gwebsocket.Conn, period time.Duration, deadline time.Duration) *heartbeat {
+	h := &heartbeat{
+		conn:   conn,
+		period: period,
+		closer: make(chan struct{}),
+	}
+	// Set up handler for receiving returned "pong" message from other endpoint
+	// by pushing the read deadline into the future. The "msg" received could
+	// be empty.
+	h.conn.SetPongHandler(func(msg string) error {
+		// Push the read deadline into the future.
+		klog.V(8).Infof("Pong message received (%s)--resetting read deadline", msg)
+		err := h.conn.SetReadDeadline(time.Now().Add(deadline))
+		if err != nil {
+			klog.Errorf("Websocket setting read deadline failed %v", err)
+			return err
+		}
+		if len(msg) > 0 {
+			h.pongMessage = []byte(msg)
+		}
+		return nil
+	})
+	// Set up handler to cleanup timers when this endpoint receives "Close" message.
+	closeHandler := h.conn.CloseHandler()
+	h.conn.SetCloseHandler(func(code int, text string) error {
+		close(h.closer)
+		return closeHandler(code, text)
+	})
+	return h
+}
+
+// setMessage is optional data sent with "ping" heartbeat. According to the websocket RFC
+// this data sent with "ping" message should be returned in "pong" message.
+func (h *heartbeat) setMessage(msg string) {
+	h.message = []byte(msg)
+}
+
+// start the heartbeat by setting up necesssary handlers and looping by sending "ping"
+// message every "period" until the "closer" channel is closed.
+func (h *heartbeat) start() {
+	// Loop to continually send "ping" message through websocket connection every "period".
+	t := time.NewTicker(h.period)
+	defer t.Stop()
+	for {
+		select {
+		case <-h.closer:
+			klog.V(8).Infof("closed channel--returning")
+			return
+		case <-t.C:
+			// "WriteControl" does not need to be protected by a mutex. According to
+			// gorilla/websockets library docs: "The Close and WriteControl methods can
+			// be called concurrently with all other methods."
+			if err := h.conn.WriteControl(gwebsocket.PingMessage, h.message, time.Now().Add(pingReadDeadline)); err == nil {
+				klog.V(8).Infof("Websocket Ping succeeeded")
+			} else {
+				klog.Errorf("Websocket Ping failed: %v", err)
+				if errors.Is(err, gwebsocket.ErrCloseSent) {
+					// we continue because c.conn.CloseChan will manage closing the connection already
+					continue
+				} else if e, ok := err.(net.Error); ok && e.Timeout() {
+					// Continue, in case this is a transient failure.
+					// c.conn.CloseChan above will tell us when the connection is
+					// actually closed.
+					// If Temporary function hadn't been deprecated, we would have used it.
+					// But most of temporary errors are timeout errors anyway.
+					continue
+				}
+				return
+			}
+		}
+	}
+}