Changes to accommodate client-go changes and kube vendor update

to v1.18.0

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal
2020-04-14 12:34:33 +05:30
committed by mergify[bot]
parent 4c96ad3c85
commit 34fc1d847e
1083 changed files with 50505 additions and 155846 deletions

View File

@ -0,0 +1,232 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package egressselector
import (
"fmt"
"io/ioutil"
"strings"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/apis/apiserver"
"k8s.io/apiserver/pkg/apis/apiserver/install"
"k8s.io/apiserver/pkg/apis/apiserver/v1beta1"
"k8s.io/utils/path"
"sigs.k8s.io/yaml"
)
var cfgScheme = runtime.NewScheme()
func init() {
install.Install(cfgScheme)
}
// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path.
// It returns the loaded egress selector configuration if the input file aligns with the required syntax.
// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op.
// It does this by returning a nil configuration, which preserves backward compatibility.
// This works because prior to this there was no egress selector configuration.
// It returns an error if the file did not exist.
func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) {
if configFilePath == "" {
return nil, nil
}
// a file was provided, so we just read it.
data, err := ioutil.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
}
var decodedConfig v1beta1.EgressSelectorConfiguration
err = yaml.Unmarshal(data, &decodedConfig)
if err != nil {
// we got an error where the decode wasn't related to a missing type
return nil, err
}
if decodedConfig.Kind != "EgressSelectorConfiguration" {
return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind)
}
internalConfig := &apiserver.EgressSelectorConfiguration{}
if err := cfgScheme.Convert(&decodedConfig, internalConfig, nil); err != nil {
// we got an error where the decode wasn't related to a missing type
return nil, err
}
return internalConfig, nil
}
// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for
// common configuration errors. It will return error for problems such as configuring mtls/cert
// settings for protocol which do not support security. It will also try to catch errors such as
// incorrect file paths. It will return nil if it does not find anything wrong.
func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList {
allErrs := field.ErrorList{}
if config == nil {
return allErrs // Treating a nil configuration as valid
}
for _, service := range config.EgressSelections {
fldPath := field.NewPath("service", "connection")
switch service.Connection.ProxyProtocol {
case apiserver.ProtocolDirect:
allErrs = append(allErrs, validateDirectConnection(service.Connection, fldPath)...)
case apiserver.ProtocolHTTPConnect:
allErrs = append(allErrs, validateHTTPConnectTransport(service.Connection.Transport, fldPath)...)
case apiserver.ProtocolGRPC:
allErrs = append(allErrs, validateGRPCTransport(service.Connection.Transport, fldPath)...)
default:
allErrs = append(allErrs, field.NotSupported(
fldPath.Child("protocol"),
service.Connection.ProxyProtocol,
[]string{
string(apiserver.ProtocolDirect),
string(apiserver.ProtocolHTTPConnect),
string(apiserver.ProtocolGRPC),
}))
}
}
return allErrs
}
func validateHTTPConnectTransport(transport *apiserver.Transport, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if transport == nil {
allErrs = append(allErrs, field.Required(
fldPath.Child("transport"),
"transport must be set for HTTPConnect"))
return allErrs
}
if transport.TCP != nil && transport.UDS != nil {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tcp"),
transport.TCP,
"TCP and UDS cannot both be set"))
} else if transport.TCP == nil && transport.UDS == nil {
allErrs = append(allErrs, field.Required(
fldPath.Child("tcp"),
"One of TCP or UDS must be set"))
} else if transport.TCP != nil {
allErrs = append(allErrs, validateTCPConnection(transport.TCP, fldPath)...)
} else if transport.UDS != nil {
allErrs = append(allErrs, validateUDSConnection(transport.UDS, fldPath)...)
}
return allErrs
}
func validateGRPCTransport(transport *apiserver.Transport, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if transport == nil {
allErrs = append(allErrs, field.Required(
fldPath.Child("transport"),
"transport must be set for GRPC"))
return allErrs
}
if transport.UDS != nil {
allErrs = append(allErrs, validateUDSConnection(transport.UDS, fldPath)...)
} else {
allErrs = append(allErrs, field.Required(
fldPath.Child("uds"),
"UDS must be set with GRPC"))
}
return allErrs
}
func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
if connection.Transport != nil {
return field.ErrorList{field.Invalid(
fldPath.Child("transport"),
"direct",
"Transport config should be absent for direct connect"),
}
}
return nil
}
func validateUDSConnection(udsConfig *apiserver.UDSTransport, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if udsConfig.UDSName == "" {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("udsName"),
"nil",
"UDSName should be present for UDS connections"))
}
return allErrs
}
func validateTCPConnection(tcpConfig *apiserver.TCPTransport, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if strings.HasPrefix(tcpConfig.URL, "http://") {
if tcpConfig.TLSConfig != nil {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig"),
"nil",
"TLSConfig config should not be present when using HTTP"))
}
} else if strings.HasPrefix(tcpConfig.URL, "https://") {
return validateTLSConfig(tcpConfig.TLSConfig, fldPath)
} else {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("url"),
tcpConfig.URL,
"supported connection protocols are http:// and https://"))
}
return allErrs
}
func validateTLSConfig(tlsConfig *apiserver.TLSConfig, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if tlsConfig == nil {
allErrs = append(allErrs, field.Required(
fldPath.Child("tlsConfig"),
"TLSConfig must be present when using HTTPS"))
return allErrs
}
if tlsConfig.CABundle != "" {
if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.CABundle); exists == false || err != nil {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig", "caBundle"),
tlsConfig.CABundle,
"TLS config ca bundle does not exist"))
}
}
if tlsConfig.ClientCert == "" {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig", "clientCert"),
"nil",
"Using TLS requires clientCert"))
} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientCert); exists == false || err != nil {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig", "clientCert"),
tlsConfig.ClientCert,
"TLS client cert does not exist"))
}
if tlsConfig.ClientKey == "" {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig", "clientKey"),
"nil",
"Using TLS requires requires clientKey"))
} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientKey); exists == false || err != nil {
allErrs = append(allErrs, field.Invalid(
fldPath.Child("tlsConfig", "clientKey"),
tlsConfig.ClientKey,
"TLS client key does not exist"))
}
return allErrs
}

View File

@ -0,0 +1,368 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package egressselector
import (
"bufio"
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"strings"
"time"
"google.golang.org/grpc"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/apis/apiserver"
egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
"k8s.io/klog"
utiltrace "k8s.io/utils/trace"
client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
)
var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
// EgressSelector is the map of network context type to context dialer, for network egress.
type EgressSelector struct {
egressToDialer map[EgressType]utilnet.DialFunc
}
// EgressType is an indicator of which egress selection should be used for sending traffic.
// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context
type EgressType int
const (
// Master is the EgressType for traffic intended to go to the control plane.
Master EgressType = iota
// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
Etcd
// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
Cluster
)
// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
type NetworkContext struct {
// EgressSelectionName is the unique name of the
// EgressSelectorConfiguration which determines
// the network we route the traffic to.
EgressSelectionName EgressType
}
// Lookup is the interface to get the dialer function for the network context.
type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
// String returns the canonical string representation of the egress type
func (s EgressType) String() string {
switch s {
case Master:
return "master"
case Etcd:
return "etcd"
case Cluster:
return "cluster"
default:
return "invalid"
}
}
// AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
func (s EgressType) AsNetworkContext() NetworkContext {
return NetworkContext{EgressSelectionName: s}
}
func lookupServiceName(name string) (EgressType, error) {
switch strings.ToLower(name) {
case "master":
return Master, nil
case "etcd":
return Etcd, nil
case "cluster":
return Cluster, nil
}
return -1, fmt.Errorf("unrecognized service name %s", name)
}
func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) {
fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
br := bufio.NewReader(proxyConn)
res, err := http.ReadResponse(br, nil)
if err != nil {
proxyConn.Close()
return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
addr, proxyAddress, err)
}
if res.StatusCode != 200 {
proxyConn.Close()
return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
proxyAddress, addr, res.StatusCode, res.Status)
}
// It's safe to discard the bufio.Reader here and return the
// original TCP conn directly because we only use this for
// TLS, and in TLS the client speaks first, so we know there's
// no unbuffered data. But we can double-check.
if br.Buffered() > 0 {
proxyConn.Close()
return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
br.Buffered(), proxyAddress)
}
return proxyConn, nil
}
type proxier interface {
// proxy returns a connection to addr.
proxy(addr string) (net.Conn, error)
}
var _ proxier = &httpConnectProxier{}
type httpConnectProxier struct {
conn net.Conn
proxyAddress string
}
func (t *httpConnectProxier) proxy(addr string) (net.Conn, error) {
return tunnelHTTPConnect(t.conn, t.proxyAddress, addr)
}
var _ proxier = &grpcProxier{}
type grpcProxier struct {
tunnel client.Tunnel
}
func (g *grpcProxier) proxy(addr string) (net.Conn, error) {
return g.tunnel.Dial("tcp", addr)
}
type proxyServerConnector interface {
// connect establishes connection to the proxy server, and returns a
// proxier based on the connection.
connect() (proxier, error)
}
type tcpHTTPConnectConnector struct {
proxyAddress string
tlsConfig *tls.Config
}
func (t *tcpHTTPConnectConnector) connect() (proxier, error) {
conn, err := tls.Dial("tcp", t.proxyAddress, t.tlsConfig)
if err != nil {
return nil, err
}
return &httpConnectProxier{conn: conn, proxyAddress: t.proxyAddress}, nil
}
type udsHTTPConnectConnector struct {
udsName string
}
func (u *udsHTTPConnectConnector) connect() (proxier, error) {
conn, err := net.Dial("unix", u.udsName)
if err != nil {
return nil, err
}
return &httpConnectProxier{conn: conn, proxyAddress: u.udsName}, nil
}
type udsGRPCConnector struct {
udsName string
}
func (u *udsGRPCConnector) connect() (proxier, error) {
udsName := u.udsName
dialOption := grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) {
c, err := net.Dial("unix", udsName)
if err != nil {
klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err)
}
return c, err
})
tunnel, err := client.CreateGrpcTunnel(udsName, dialOption, grpc.WithInsecure())
if err != nil {
return nil, err
}
return &grpcProxier{tunnel: tunnel}, nil
}
type dialerCreator struct {
connector proxyServerConnector
direct bool
options metricsOptions
}
type metricsOptions struct {
transport string
protocol string
}
func (d *dialerCreator) createDialer() utilnet.DialFunc {
if d.direct {
return directDialer
}
return func(ctx context.Context, network, addr string) (net.Conn, error) {
trace := utiltrace.New(fmt.Sprintf("Proxy via HTTP Connect over %s", d.options.transport), utiltrace.Field{Key: "address", Value: addr})
defer trace.LogIfLong(500 * time.Millisecond)
start := egressmetrics.Metrics.Clock().Now()
proxier, err := d.connector.connect()
if err != nil {
egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect)
return nil, err
}
conn, err := proxier.proxy(addr)
if err != nil {
egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageProxy)
return nil, err
}
egressmetrics.Metrics.ObserveDialLatency(egressmetrics.Metrics.Clock().Now().Sub(start), d.options.protocol, d.options.transport)
return conn, nil
}
}
func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) {
clientCert := t.ClientCert
clientKey := t.ClientKey
caCert := t.CABundle
clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
}
certPool := x509.NewCertPool()
if caCert != "" {
certBytes, err := ioutil.ReadFile(caCert)
if err != nil {
return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
}
ok := certPool.AppendCertsFromPEM(certBytes)
if !ok {
return nil, fmt.Errorf("failed to append CA cert to the cert pool")
}
} else {
// Use host's root CA set instead of providing our own
certPool = nil
}
return &tls.Config{
Certificates: []tls.Certificate{clientCerts},
RootCAs: certPool,
}, nil
}
func getProxyAddress(urlString string) (string, error) {
proxyURL, err := url.Parse(urlString)
if err != nil {
return "", fmt.Errorf("invalid proxy server url %q: %v", urlString, err)
}
return proxyURL.Host, nil
}
func connectionToDialerCreator(c apiserver.Connection) (*dialerCreator, error) {
switch c.ProxyProtocol {
case apiserver.ProtocolHTTPConnect:
if c.Transport.UDS != nil {
return &dialerCreator{
connector: &udsHTTPConnectConnector{
udsName: c.Transport.UDS.UDSName,
},
options: metricsOptions{
transport: egressmetrics.TransportUDS,
protocol: egressmetrics.ProtocolHTTPConnect,
},
}, nil
} else if c.Transport.TCP != nil {
tlsConfig, err := getTLSConfig(c.Transport.TCP.TLSConfig)
if err != nil {
return nil, err
}
proxyAddress, err := getProxyAddress(c.Transport.TCP.URL)
if err != nil {
return nil, err
}
return &dialerCreator{
connector: &tcpHTTPConnectConnector{
tlsConfig: tlsConfig,
proxyAddress: proxyAddress,
},
options: metricsOptions{
transport: egressmetrics.TransportTCP,
protocol: egressmetrics.ProtocolHTTPConnect,
},
}, nil
} else {
return nil, fmt.Errorf("Either a TCP or UDS transport must be specified")
}
case apiserver.ProtocolGRPC:
if c.Transport.UDS != nil {
return &dialerCreator{
connector: &udsGRPCConnector{
udsName: c.Transport.UDS.UDSName,
},
options: metricsOptions{
transport: egressmetrics.TransportUDS,
protocol: egressmetrics.ProtocolGRPC,
},
}, nil
}
return nil, fmt.Errorf("UDS transport must be specified for GRPC")
case apiserver.ProtocolDirect:
return &dialerCreator{direct: true}, nil
default:
return nil, fmt.Errorf("unrecognized service connection protocol %q", c.ProxyProtocol)
}
}
// NewEgressSelector configures lookup mechanism for Lookup.
// It does so based on a EgressSelectorConfiguration which was read at startup.
func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
if config == nil || config.EgressSelections == nil {
// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
return nil, nil
}
cs := &EgressSelector{
egressToDialer: make(map[EgressType]utilnet.DialFunc),
}
for _, service := range config.EgressSelections {
name, err := lookupServiceName(service.Name)
if err != nil {
return nil, err
}
dialerCreator, err := connectionToDialerCreator(service.Connection)
if err != nil {
return nil, fmt.Errorf("failed to create dialer for egressSelection %q: %v", name, err)
}
cs.egressToDialer[name] = dialerCreator.createDialer()
}
return cs, nil
}
// Lookup gets the dialer function for the network context.
// This is configured for the Kubernetes API Server at startup.
func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
if cs.egressToDialer == nil {
// The round trip wrapper will over-ride the dialContext method appropriately
return nil, nil
}
return cs.egressToDialer[networkContext.EgressSelectionName], nil
}

View File

@ -0,0 +1,114 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"time"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
const (
namespace = "apiserver"
subsystem = "egress_dialer"
// ProtocolHTTPConnect means that the proxy protocol is http-connect.
ProtocolHTTPConnect = "http_connect"
// ProtocolGRPC means that the proxy protocol is the GRPC protocol.
ProtocolGRPC = "grpc"
// TransportTCP means that the transport is TCP.
TransportTCP = "tcp"
// TransportUDS means that the transport is UDS.
TransportUDS = "uds"
// StageConnect indicates that the dial failed at establishing connection to the proxy server.
StageConnect = "connect"
// StageProxy indicates that the dial failed at requesting the proxy server to proxy.
StageProxy = "proxy"
)
var (
// Use buckets ranging from 5 ms to 12.5 seconds.
latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 2.5, 12.5}
// Metrics provides access to all dial metrics.
Metrics = newDialMetrics()
)
// DialMetrics instruments dials to proxy server with prometheus metrics.
type DialMetrics struct {
clock clock.Clock
latencies *metrics.HistogramVec
failures *metrics.CounterVec
}
// newDialMetrics create a new DialMetrics, configured with default metric names.
func newDialMetrics() *DialMetrics {
latencies := metrics.NewHistogramVec(
&metrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "dial_duration_seconds",
Help: "Dial latency histogram in seconds, labeled by the protocol (http-connect or grpc), transport (tcp or uds)",
Buckets: latencyBuckets,
StabilityLevel: metrics.ALPHA,
},
[]string{"protocol", "transport"},
)
failures := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "dial_failure_count",
Help: "Dial failure count, labeled by the protocol (http-connect or grpc), transport (tcp or uds), and stage (connect or proxy). The stage indicates at which stage the dial failed",
StabilityLevel: metrics.ALPHA,
},
[]string{"protocol", "transport", "stage"},
)
legacyregistry.MustRegister(latencies)
legacyregistry.MustRegister(failures)
return &DialMetrics{latencies: latencies, failures: failures, clock: clock.RealClock{}}
}
// Clock returns the clock.
func (m *DialMetrics) Clock() clock.Clock {
return m.clock
}
// SetClock sets the clock.
func (m *DialMetrics) SetClock(c clock.Clock) {
m.clock = c
}
// Reset resets the metrics.
func (m *DialMetrics) Reset() {
m.latencies.Reset()
m.failures.Reset()
}
// ObserveDialLatency records the latency of a dial, labeled by protocol, transport.
func (m *DialMetrics) ObserveDialLatency(elapsed time.Duration, protocol, transport string) {
m.latencies.WithLabelValues(protocol, transport).Observe(elapsed.Seconds())
}
// ObserveDialFailure records a failed dial, labeled by protocol, transport, and the stage the dial failed at.
func (m *DialMetrics) ObserveDialFailure(protocol, transport, stage string) {
m.failures.WithLabelValues(protocol, transport, stage).Inc()
}