build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>
This commit is contained in:
Niels de Vos
2025-03-04 08:57:28 +01:00
committed by mergify[bot]
parent 15da101b1b
commit bec6090996
8047 changed files with 1407827 additions and 3453 deletions

View File

@ -0,0 +1,264 @@
//go:build windows
// +build windows
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package winstats
import (
"fmt"
"syscall"
"unsafe"
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/klog/v2"
)
var (
procGetLogicalProcessorInformationEx = modkernel32.NewProc("GetLogicalProcessorInformationEx")
getNumaAvailableMemoryNodeEx = modkernel32.NewProc("GetNumaAvailableMemoryNodeEx")
procGetNumaNodeProcessorMaskEx = modkernel32.NewProc("GetNumaNodeProcessorMaskEx")
)
type relationType int
const (
relationProcessorCore relationType = iota
relationNumaNode
relationCache
relationProcessorPackage
relationGroup
relationProcessorDie
relationNumaNodeEx
relationProcessorModule
relationAll = 0xffff
)
type systemLogicalProcessorInformationEx struct {
Relationship uint32
Size uint32
data interface{}
}
type processorRelationship struct {
Flags byte
EfficiencyClass byte
Reserved [20]byte
GroupCount uint16
// groupMasks is an []GroupAffinity. In c++ this is a union of either one or many GroupAffinity based on GroupCount
GroupMasks interface{}
}
// GroupAffinity represents the processor group affinity of cpus
// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-group_affinity
type GroupAffinity struct {
Mask uint64
Group uint16
Reserved [3]uint16
}
// MaskString returns the affinity mask as a string of 0s and 1s
func (a GroupAffinity) MaskString() string {
return fmt.Sprintf("%064b", a.Mask)
}
// Processors returns a list of processors ids that are part of the affinity mask
// Windows doesn't track processors by ID but kubelet converts them to a number
func (a GroupAffinity) Processors() []int {
processors := []int{}
for i := 0; i < 64; i++ {
if a.Mask&(1<<i) != 0 {
processors = append(processors, i+(int(a.Group)*64))
}
}
return processors
}
// CpusToGroupAffinity converts a list of CPUs to a map of GroupAffinity split by windows CPU group.
// Windows doesn't track processors by ID but kubelet converts them to a number and this function goes in reverse.
func CpusToGroupAffinity(cpus []int) map[int]*GroupAffinity {
groupAffinities := make(map[int]*GroupAffinity)
for _, cpu := range cpus {
group := uint16(cpu / 64)
groupAffinity, ok := groupAffinities[int(group)]
if !ok {
groupAffinity = &GroupAffinity{
Group: group,
}
groupAffinities[int(group)] = groupAffinity
}
mask := uint64(1 << (cpu % 64))
groupAffinity.Mask |= mask
}
return groupAffinities
}
// GetCPUsForNUMANode queries the system for the CPUs that are part of the given NUMA node.
func GetCPUsforNUMANode(nodeNumber uint16) (*GroupAffinity, error) {
var affinity GroupAffinity
r1, _, err := procGetNumaNodeProcessorMaskEx.Call(
uintptr(nodeNumber),
uintptr(unsafe.Pointer(&affinity)),
)
if r1 == 0 {
return nil, fmt.Errorf("Error getting CPU mask for NUMA node %d: %v", nodeNumber, err)
}
return &affinity, nil
}
type numaNodeRelationship struct {
NodeNumber uint32
Reserved [18]byte
GroupCount uint16
GroupMasks interface{} //[]GroupAffinity in c++ this is a union of either one or many GroupAffinity based on GroupCount
}
type processor struct {
CoreID int
SocketID int
NodeID int
}
func processorInfo(relationShip relationType) (int, int, []cadvisorapi.Node, error) {
// Call once to get the length of data to return
var returnLength uint32 = 0
r1, _, err := procGetLogicalProcessorInformationEx.Call(
uintptr(relationShip),
uintptr(0),
uintptr(unsafe.Pointer(&returnLength)),
)
if r1 != 0 && err.(syscall.Errno) != syscall.ERROR_INSUFFICIENT_BUFFER {
return 0, 0, nil, fmt.Errorf("call to GetLogicalProcessorInformationEx failed: %v", err)
}
// Allocate the buffer with the length it should be
buffer := make([]byte, returnLength)
// Call GetLogicalProcessorInformationEx again to get the actual information
r1, _, err = procGetLogicalProcessorInformationEx.Call(
uintptr(relationShip),
uintptr(unsafe.Pointer(&buffer[0])),
uintptr(unsafe.Pointer(&returnLength)),
)
if r1 == 0 {
return 0, 0, nil, fmt.Errorf("call to GetLogicalProcessorInformationEx failed: %v", err)
}
return convertWinApiToCadvisorApi(buffer)
}
func convertWinApiToCadvisorApi(buffer []byte) (int, int, []cadvisorapi.Node, error) {
logicalProcessors := make(map[int]*processor)
numofSockets := 0
numOfcores := 0
nodes := []cadvisorapi.Node{}
for offset := 0; offset < len(buffer); {
// check size in buffer to avoid out of bounds access, we don't know the type or size yet
if offset+int(unsafe.Sizeof(systemLogicalProcessorInformationEx{})) > len(buffer) {
return 0, 0, nil, fmt.Errorf("remaining buffer too small while reading windows processor relationship")
}
info := (*systemLogicalProcessorInformationEx)(unsafe.Pointer(&buffer[offset]))
// check one more time now that we know the size of the struct
if offset+int(info.Size) > len(buffer) {
return 0, 0, nil, fmt.Errorf("remaining buffer too small while reading windows processor relationship")
}
switch (relationType)(info.Relationship) {
case relationProcessorCore, relationProcessorPackage:
relationship := (*processorRelationship)(unsafe.Pointer(&info.data))
groupMasks := make([]GroupAffinity, relationship.GroupCount)
for i := 0; i < int(relationship.GroupCount); i++ {
groupMasks[i] = *(*GroupAffinity)(unsafe.Pointer(uintptr(unsafe.Pointer(&relationship.GroupMasks)) + uintptr(i)*unsafe.Sizeof(GroupAffinity{})))
}
if relationProcessorCore == (relationType)(info.Relationship) {
numOfcores++
}
if relationProcessorPackage == (relationType)(info.Relationship) {
numofSockets++
}
//iterate over group masks and add each processor to the map
for _, groupMask := range groupMasks {
for _, processorId := range groupMask.Processors() {
p, ok := logicalProcessors[processorId]
if !ok {
p = &processor{}
logicalProcessors[processorId] = p
}
if relationProcessorCore == (relationType)(info.Relationship) {
p.CoreID = numOfcores
}
if relationProcessorPackage == (relationType)(info.Relationship) {
p.SocketID = numofSockets
}
}
}
case relationNumaNode, relationNumaNodeEx:
numaNodeRelationship := (*numaNodeRelationship)(unsafe.Pointer(&info.data))
groupMasks := make([]GroupAffinity, numaNodeRelationship.GroupCount)
for i := 0; i < int(numaNodeRelationship.GroupCount); i++ {
groupMasks[i] = *(*GroupAffinity)(unsafe.Pointer(uintptr(unsafe.Pointer(&numaNodeRelationship.GroupMasks)) + uintptr(i)*unsafe.Sizeof(GroupAffinity{})))
}
nodes = append(nodes, cadvisorapi.Node{Id: int(numaNodeRelationship.NodeNumber)})
for _, groupMask := range groupMasks {
for processorId := range groupMask.Processors() {
p, ok := logicalProcessors[processorId]
if !ok {
p = &processor{}
logicalProcessors[processorId] = p
}
p.NodeID = int(numaNodeRelationship.NodeNumber)
}
}
default:
klog.V(4).Infof("Not using Windows CPU relationship type: %d", info.Relationship)
}
// Move the offset to the next SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX struct
offset += int(info.Size)
}
for processId, p := range logicalProcessors {
node := nodes[p.NodeID]
if node.Id != p.NodeID {
return 0, 0, nil, fmt.Errorf("node ID mismatch: %d != %d", node.Id, p.NodeID)
}
availableBytes := uint64(0)
r1, _, err := getNumaAvailableMemoryNodeEx.Call(uintptr(p.NodeID), uintptr(unsafe.Pointer(&availableBytes)))
if r1 == 0 {
return 0, 0, nil, fmt.Errorf("call to GetNumaAvailableMemoryNodeEx failed: %v", err)
}
node.Memory = availableBytes
node.AddThread(processId, p.CoreID)
ok, coreIdx := node.FindCore(p.CoreID)
if !ok {
return 0, 0, nil, fmt.Errorf("core not found: %d", p.CoreID)
}
node.Cores[coreIdx].SocketID = p.SocketID
nodes[p.NodeID] = node
}
return numOfcores, numofSockets, nodes, nil
}

View File

@ -0,0 +1,18 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package winstats provides a client to get node and pod level stats on windows
package winstats // import "k8s.io/kubernetes/pkg/kubelet/winstats"

View File

@ -0,0 +1,313 @@
//go:build windows
// +build windows
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package winstats
import (
"sync"
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
)
const (
packetsReceivedPerSecondQuery = "\\Network Adapter(*)\\Packets Received/sec"
packetsSentPerSecondQuery = "\\Network Adapter(*)\\Packets Sent/sec"
bytesReceivedPerSecondQuery = "\\Network Adapter(*)\\Bytes Received/sec"
bytesSentPerSecondQuery = "\\Network Adapter(*)\\Bytes Sent/sec"
packetsReceivedDiscardedQuery = "\\Network Adapter(*)\\Packets Received Discarded"
packetsReceivedErrorsQuery = "\\Network Adapter(*)\\Packets Received Errors"
packetsOutboundDiscardedQuery = "\\Network Adapter(*)\\Packets Outbound Discarded"
packetsOutboundErrorsQuery = "\\Network Adapter(*)\\Packets Outbound Errors"
)
// networkCounter contains the counters for network adapters.
type networkCounter struct {
packetsReceivedPerSecondCounter perfCounter
packetsSentPerSecondCounter perfCounter
bytesReceivedPerSecondCounter perfCounter
bytesSentPerSecondCounter perfCounter
packetsReceivedDiscardedCounter perfCounter
packetsReceivedErrorsCounter perfCounter
packetsOutboundDiscardedCounter perfCounter
packetsOutboundErrorsCounter perfCounter
mu sync.RWMutex
adapterStats map[string]cadvisorapi.InterfaceStats
}
func newNetworkCounters() (*networkCounter, error) {
packetsReceivedPerSecondCounter, err := newPerfCounter(packetsReceivedPerSecondQuery)
if err != nil {
return nil, err
}
packetsSentPerSecondCounter, err := newPerfCounter(packetsSentPerSecondQuery)
if err != nil {
return nil, err
}
bytesReceivedPerSecondCounter, err := newPerfCounter(bytesReceivedPerSecondQuery)
if err != nil {
return nil, err
}
bytesSentPerSecondCounter, err := newPerfCounter(bytesSentPerSecondQuery)
if err != nil {
return nil, err
}
packetsReceivedDiscardedCounter, err := newPerfCounter(packetsReceivedDiscardedQuery)
if err != nil {
return nil, err
}
packetsReceivedErrorsCounter, err := newPerfCounter(packetsReceivedErrorsQuery)
if err != nil {
return nil, err
}
packetsOutboundDiscardedCounter, err := newPerfCounter(packetsOutboundDiscardedQuery)
if err != nil {
return nil, err
}
packetsOutboundErrorsCounter, err := newPerfCounter(packetsOutboundErrorsQuery)
if err != nil {
return nil, err
}
return &networkCounter{
packetsReceivedPerSecondCounter: packetsReceivedPerSecondCounter,
packetsSentPerSecondCounter: packetsSentPerSecondCounter,
bytesReceivedPerSecondCounter: bytesReceivedPerSecondCounter,
bytesSentPerSecondCounter: bytesSentPerSecondCounter,
packetsReceivedDiscardedCounter: packetsReceivedDiscardedCounter,
packetsReceivedErrorsCounter: packetsReceivedErrorsCounter,
packetsOutboundDiscardedCounter: packetsOutboundDiscardedCounter,
packetsOutboundErrorsCounter: packetsOutboundErrorsCounter,
adapterStats: map[string]cadvisorapi.InterfaceStats{},
}, nil
}
func (n *networkCounter) getData() ([]cadvisorapi.InterfaceStats, error) {
packetsReceivedPerSecondData, err := n.packetsReceivedPerSecondCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsReceivedPerSecond perf counter data")
return nil, err
}
packetsSentPerSecondData, err := n.packetsSentPerSecondCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsSentPerSecond perf counter data")
return nil, err
}
bytesReceivedPerSecondData, err := n.bytesReceivedPerSecondCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get bytesReceivedPerSecond perf counter data")
return nil, err
}
bytesSentPerSecondData, err := n.bytesSentPerSecondCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get bytesSentPerSecond perf counter data")
return nil, err
}
packetsReceivedDiscardedData, err := n.packetsReceivedDiscardedCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsReceivedDiscarded perf counter data")
return nil, err
}
packetsReceivedErrorsData, err := n.packetsReceivedErrorsCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsReceivedErrors perf counter data")
return nil, err
}
packetsOutboundDiscardedData, err := n.packetsOutboundDiscardedCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsOutboundDiscarded perf counter data")
return nil, err
}
packetsOutboundErrorsData, err := n.packetsOutboundErrorsCounter.getDataList()
if err != nil {
klog.ErrorS(err, "Unable to get packetsOutboundErrors perf counter data")
return nil, err
}
n.mu.Lock()
defer n.mu.Unlock()
n.mergeCollectedData(
packetsReceivedPerSecondData,
packetsSentPerSecondData,
bytesReceivedPerSecondData,
bytesSentPerSecondData,
packetsReceivedDiscardedData,
packetsReceivedErrorsData,
packetsOutboundDiscardedData,
packetsOutboundErrorsData,
)
return n.listInterfaceStats(), nil
}
// mergeCollectedData merges the collected data into cache. It should be invoked under lock protected.
func (n *networkCounter) mergeCollectedData(packetsReceivedPerSecondData,
packetsSentPerSecondData,
bytesReceivedPerSecondData,
bytesSentPerSecondData,
packetsReceivedDiscardedData,
packetsReceivedErrorsData,
packetsOutboundDiscardedData,
packetsOutboundErrorsData map[string]uint64) {
adapters := sets.New[string]()
// merge the collected data and list of adapters.
adapters.Insert(n.mergePacketsReceivedPerSecondData(packetsReceivedPerSecondData)...)
adapters.Insert(n.mergePacketsSentPerSecondData(packetsSentPerSecondData)...)
adapters.Insert(n.mergeBytesReceivedPerSecondData(bytesReceivedPerSecondData)...)
adapters.Insert(n.mergeBytesSentPerSecondData(bytesSentPerSecondData)...)
adapters.Insert(n.mergePacketsReceivedDiscardedData(packetsReceivedDiscardedData)...)
adapters.Insert(n.mergePacketsReceivedErrorsData(packetsReceivedErrorsData)...)
adapters.Insert(n.mergePacketsOutboundDiscardedData(packetsOutboundDiscardedData)...)
adapters.Insert(n.mergePacketsOutboundErrorsData(packetsOutboundErrorsData)...)
// delete the cache for non-existing adapters.
for adapter := range n.adapterStats {
if !adapters.Has(adapter) {
delete(n.adapterStats, adapter)
}
}
}
func (n *networkCounter) mergePacketsReceivedPerSecondData(packetsReceivedPerSecondData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsReceivedPerSecondData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.RxPackets = newStat.RxPackets + value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergePacketsSentPerSecondData(packetsSentPerSecondData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsSentPerSecondData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.TxPackets = newStat.TxPackets + value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergeBytesReceivedPerSecondData(bytesReceivedPerSecondData map[string]uint64) []string {
var adapters []string
for adapterName, value := range bytesReceivedPerSecondData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.RxBytes = newStat.RxBytes + value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergeBytesSentPerSecondData(bytesSentPerSecondData map[string]uint64) []string {
var adapters []string
for adapterName, value := range bytesSentPerSecondData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.TxBytes = newStat.TxBytes + value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergePacketsReceivedDiscardedData(packetsReceivedDiscardedData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsReceivedDiscardedData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.RxDropped = value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergePacketsReceivedErrorsData(packetsReceivedErrorsData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsReceivedErrorsData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.RxErrors = value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergePacketsOutboundDiscardedData(packetsOutboundDiscardedData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsOutboundDiscardedData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.TxDropped = value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) mergePacketsOutboundErrorsData(packetsOutboundErrorsData map[string]uint64) []string {
var adapters []string
for adapterName, value := range packetsOutboundErrorsData {
adapters = append(adapters, adapterName)
newStat := n.adapterStats[adapterName]
newStat.Name = adapterName
newStat.TxErrors = value
n.adapterStats[adapterName] = newStat
}
return adapters
}
func (n *networkCounter) listInterfaceStats() []cadvisorapi.InterfaceStats {
stats := make([]cadvisorapi.InterfaceStats, 0, len(n.adapterStats))
for _, stat := range n.adapterStats {
stats = append(stats, stat)
}
return stats
}

View File

@ -0,0 +1,358 @@
//go:build windows
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package winstats
import (
"os"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
"unsafe"
utilfeature "k8s.io/apiserver/pkg/util/feature"
kubefeatures "k8s.io/kubernetes/pkg/features"
cadvisorapi "github.com/google/cadvisor/info/v1"
"github.com/pkg/errors"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
)
const (
bootIdRegistry = `SYSTEM\CurrentControlSet\Control\Session Manager\Memory Management\PrefetchParameters`
bootIdKey = `BootId`
)
// MemoryStatusEx is the same as Windows structure MEMORYSTATUSEX
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366770(v=vs.85).aspx
type MemoryStatusEx struct {
Length uint32
MemoryLoad uint32
TotalPhys uint64
AvailPhys uint64
TotalPageFile uint64
AvailPageFile uint64
TotalVirtual uint64
AvailVirtual uint64
AvailExtendedVirtual uint64
}
// PerformanceInfo is the same as Windows structure PERFORMANCE_INFORMATION
// https://learn.microsoft.com/en-us/windows/win32/api/psapi/ns-psapi-performance_information
type PerformanceInformation struct {
cb uint32
CommitTotalPages uint64
CommitLimitPages uint64
CommitPeakPages uint64
PhysicalTotalPages uint64
PhysicalAvailablePages uint64
SystemCachePages uint64
KernelTotalPages uint64
KernelPagesPages uint64
KernelNonpagedPages uint64
PageSize uint64
HandleCount uint32
ProcessCount uint32
ThreadCount uint32
}
var (
// kernel32.dll system calls
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx")
procGetActiveProcessorCount = modkernel32.NewProc("GetActiveProcessorCount")
// psapi.dll system calls
modpsapi = windows.NewLazySystemDLL("psapi.dll")
procGetPerformanceInfo = modpsapi.NewProc("GetPerformanceInfo")
)
const allProcessorGroups = 0xFFFF
// NewPerfCounterClient creates a client using perf counters
func NewPerfCounterClient() (Client, error) {
// Initialize the cache
initCache := cpuUsageCoreNanoSecondsCache{0, 0}
return newClient(&perfCounterNodeStatsClient{
cpuUsageCoreNanoSecondsCache: initCache,
})
}
// perfCounterNodeStatsClient is a client that provides Windows Stats via PerfCounters
type perfCounterNodeStatsClient struct {
nodeMetrics
mu sync.RWMutex // mu protects nodeMetrics
nodeInfo
// cpuUsageCoreNanoSecondsCache caches the cpu usage for nodes.
cpuUsageCoreNanoSecondsCache
}
func (p *perfCounterNodeStatsClient) startMonitoring() error {
memory, err := getPhysicallyInstalledSystemMemoryBytes()
if err != nil {
return err
}
osInfo, err := GetOSInfo()
if err != nil {
return err
}
p.nodeInfo = nodeInfo{
kernelVersion: osInfo.GetPatchVersion(),
osImageVersion: osInfo.ProductName,
memoryPhysicalCapacityBytes: memory,
startTime: time.Now(),
}
cpuCounter, err := newPerfCounter(cpuQuery)
if err != nil {
return err
}
memWorkingSetCounter, err := newPerfCounter(memoryPrivWorkingSetQuery)
if err != nil {
return err
}
memCommittedBytesCounter, err := newPerfCounter(memoryCommittedBytesQuery)
if err != nil {
return err
}
networkAdapterCounter, err := newNetworkCounters()
if err != nil {
return err
}
go wait.Forever(func() {
p.collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter, networkAdapterCounter)
}, perfCounterUpdatePeriod)
// Cache the CPU usage every defaultCachePeriod
go wait.Forever(func() {
newValue := p.nodeMetrics.cpuUsageCoreNanoSeconds
p.mu.Lock()
defer p.mu.Unlock()
p.cpuUsageCoreNanoSecondsCache = cpuUsageCoreNanoSecondsCache{
previousValue: p.cpuUsageCoreNanoSecondsCache.latestValue,
latestValue: newValue,
}
}, defaultCachePeriod)
return nil
}
func (p *perfCounterNodeStatsClient) getMachineInfo() (*cadvisorapi.MachineInfo, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, err
}
systemUUID, err := getSystemUUID()
if err != nil {
return nil, err
}
bootId, err := getBootID()
if err != nil {
return nil, err
}
mi := &cadvisorapi.MachineInfo{
NumCores: ProcessorCount(),
MemoryCapacity: p.nodeInfo.memoryPhysicalCapacityBytes,
MachineID: hostname,
SystemUUID: systemUUID,
BootID: bootId,
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsCPUAndMemoryAffinity) {
numOfPysicalCores, numOfSockets, topology, err := processorInfo(relationAll)
if err != nil {
return nil, err
}
mi.NumPhysicalCores = numOfPysicalCores
mi.NumSockets = numOfSockets
mi.Topology = topology
}
return mi, nil
}
// ProcessorCount returns the number of logical processors on the system.
// runtime.NumCPU() will only return the information for a single Processor Group.
// Since a single group can only hold 64 logical processors, this
// means when there are more they will be divided into multiple groups.
// For the above reason, procGetActiveProcessorCount is used to get the
// cpu count for all processor groups of the windows node.
// more notes for this issue:
// same issue in moby: https://github.com/moby/moby/issues/38935#issuecomment-744638345
// solution in hcsshim: https://github.com/microsoft/hcsshim/blob/master/internal/processorinfo/processor_count.go
func ProcessorCount() int {
if amount := getActiveProcessorCount(allProcessorGroups); amount != 0 {
return int(amount)
}
return runtime.NumCPU()
}
func getActiveProcessorCount(groupNumber uint16) int {
r0, _, _ := syscall.Syscall(procGetActiveProcessorCount.Addr(), 1, uintptr(groupNumber), 0, 0)
return int(r0)
}
func (p *perfCounterNodeStatsClient) getVersionInfo() (*cadvisorapi.VersionInfo, error) {
return &cadvisorapi.VersionInfo{
KernelVersion: p.nodeInfo.kernelVersion,
ContainerOsVersion: p.nodeInfo.osImageVersion,
}, nil
}
func (p *perfCounterNodeStatsClient) getNodeMetrics() (nodeMetrics, error) {
p.mu.RLock()
defer p.mu.RUnlock()
return p.nodeMetrics, nil
}
func (p *perfCounterNodeStatsClient) getNodeInfo() nodeInfo {
return p.nodeInfo
}
func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter perfCounter, networkAdapterCounter *networkCounter) {
cpuValue, err := cpuCounter.getData()
cpuCores := ProcessorCount()
if err != nil {
klog.ErrorS(err, "Unable to get cpu perf counter data")
return
}
memWorkingSetValue, err := memWorkingSetCounter.getData()
if err != nil {
klog.ErrorS(err, "Unable to get memWorkingSet perf counter data")
return
}
memCommittedBytesValue, err := memCommittedBytesCounter.getData()
if err != nil {
klog.ErrorS(err, "Unable to get memCommittedBytes perf counter data")
return
}
networkAdapterStats, err := networkAdapterCounter.getData()
if err != nil {
klog.ErrorS(err, "Unable to get network adapter perf counter data")
return
}
p.mu.Lock()
defer p.mu.Unlock()
p.nodeMetrics = nodeMetrics{
cpuUsageCoreNanoSeconds: p.convertCPUValue(cpuCores, cpuValue),
cpuUsageNanoCores: p.getCPUUsageNanoCores(),
memoryPrivWorkingSetBytes: memWorkingSetValue,
memoryCommittedBytes: memCommittedBytesValue,
interfaceStats: networkAdapterStats,
timeStamp: time.Now(),
}
}
func (p *perfCounterNodeStatsClient) convertCPUValue(cpuCores int, cpuValue uint64) uint64 {
// This converts perf counter data which is cpu percentage for all cores into nanoseconds.
// The formula is (cpuPercentage / 100.0) * #cores * 1e+9 (nano seconds). More info here:
// https://github.com/kubernetes/heapster/issues/650
newValue := p.nodeMetrics.cpuUsageCoreNanoSeconds + uint64((float64(cpuValue)/100.0)*float64(cpuCores)*1e9)
return newValue
}
func (p *perfCounterNodeStatsClient) getCPUUsageNanoCores() uint64 {
cachePeriodSeconds := uint64(defaultCachePeriod / time.Second)
perfCounterUpdatePeriodSeconds := uint64(perfCounterUpdatePeriod / time.Second)
cpuUsageNanoCores := ((p.cpuUsageCoreNanoSecondsCache.latestValue - p.cpuUsageCoreNanoSecondsCache.previousValue) * perfCounterUpdatePeriodSeconds) / cachePeriodSeconds
return cpuUsageNanoCores
}
func getSystemUUID() (string, error) {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\HardwareConfig`, registry.QUERY_VALUE)
if err != nil {
return "", errors.Wrap(err, "failed to open registry key HKLM\\SYSTEM\\HardwareConfig")
}
defer k.Close()
uuid, _, err := k.GetStringValue("LastConfig")
if err != nil {
return "", errors.Wrap(err, "failed to read registry value LastConfig from key HKLM\\SYSTEM\\HardwareConfig")
}
uuid = strings.Trim(uuid, "{")
uuid = strings.Trim(uuid, "}")
uuid = strings.ToUpper(uuid)
return uuid, nil
}
func getPhysicallyInstalledSystemMemoryBytes() (uint64, error) {
// We use GlobalMemoryStatusEx instead of GetPhysicallyInstalledSystemMemory
// on Windows node for the following reasons:
// 1. GetPhysicallyInstalledSystemMemory retrieves the amount of physically
// installed RAM from the computer's SMBIOS firmware tables.
// https://msdn.microsoft.com/en-us/library/windows/desktop/cc300158(v=vs.85).aspx
// On some VM, it is unable to read data from SMBIOS and fails with ERROR_INVALID_DATA.
// 2. On Linux node, total physical memory is read from MemTotal in /proc/meminfo.
// GlobalMemoryStatusEx returns the amount of physical memory that is available
// for the operating system to use. The amount returned by GlobalMemoryStatusEx
// is closer in parity with Linux
// https://www.kernel.org/doc/Documentation/filesystems/proc.txt
var statex MemoryStatusEx
statex.Length = uint32(unsafe.Sizeof(statex))
ret, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&statex)))
if ret == 0 {
return 0, errors.New("unable to read physical memory")
}
return statex.TotalPhys, nil
}
func GetPerformanceInfo() (*PerformanceInformation, error) {
var pi PerformanceInformation
pi.cb = uint32(unsafe.Sizeof(pi))
ret, _, _ := procGetPerformanceInfo.Call(uintptr(unsafe.Pointer(&pi)), uintptr(pi.cb))
if ret == 0 {
return nil, errors.New("unable to read Windows performance information")
}
return &pi, nil
}
func getBootID() (string, error) {
regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, bootIdRegistry, registry.READ)
if err != nil {
return "", err
}
defer regKey.Close()
regValue, _, err := regKey.GetIntegerValue(bootIdKey)
if err != nil {
return "", err
}
return strconv.FormatUint(regValue, 10), nil
}

View File

@ -0,0 +1,136 @@
//go:build windows
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package winstats
import (
"errors"
"fmt"
"time"
"unsafe"
"github.com/JeffAshton/win_pdh"
)
const (
cpuQuery = "\\Processor(_Total)\\% Processor Time"
memoryPrivWorkingSetQuery = "\\Process(_Total)\\Working Set - Private"
memoryCommittedBytesQuery = "\\Memory\\Committed Bytes"
// Perf counters are updated 10 seconds. This is the same as the default cadvisor housekeeping interval
// set at https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/cadvisor/cadvisor_linux.go
perfCounterUpdatePeriod = 10 * time.Second
// defaultCachePeriod is the default cache period for each cpuUsage.
// This matches with the cadvisor setting and the time interval we use for containers.
// see https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/cadvisor/cadvisor_linux.go#L63
defaultCachePeriod = 10 * time.Second
)
type perfCounter interface {
getData() (uint64, error)
getDataList() (map[string]uint64, error)
}
type perfCounterImpl struct {
queryHandle win_pdh.PDH_HQUERY
counterHandle win_pdh.PDH_HCOUNTER
}
func newPerfCounter(counter string) (perfCounter, error) {
var queryHandle win_pdh.PDH_HQUERY
var counterHandle win_pdh.PDH_HCOUNTER
ret := win_pdh.PdhOpenQuery(0, 0, &queryHandle)
if ret != win_pdh.ERROR_SUCCESS {
return nil, errors.New("unable to open query through DLL call")
}
ret = win_pdh.PdhAddEnglishCounter(queryHandle, counter, 0, &counterHandle)
if ret != win_pdh.ERROR_SUCCESS {
return nil, fmt.Errorf("unable to add process counter: %s. Error code is %x", counter, ret)
}
ret = win_pdh.PdhCollectQueryData(queryHandle)
if ret != win_pdh.ERROR_SUCCESS {
return nil, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
}
return &perfCounterImpl{
queryHandle: queryHandle,
counterHandle: counterHandle,
}, nil
}
// getData is used for getting data without * in counter name.
func (p *perfCounterImpl) getData() (uint64, error) {
filledBuf, bufCount, err := p.getQueriedData()
if err != nil {
return 0, err
}
var data uint64 = 0
for i := 0; i < int(bufCount); i++ {
c := filledBuf[i]
data = uint64(c.FmtValue.DoubleValue)
}
return data, nil
}
// getDataList is used for getting data with * in counter name.
func (p *perfCounterImpl) getDataList() (map[string]uint64, error) {
filledBuf, bufCount, err := p.getQueriedData()
if err != nil {
return nil, err
}
data := map[string]uint64{}
for i := 0; i < int(bufCount); i++ {
c := filledBuf[i]
value := uint64(c.FmtValue.DoubleValue)
name := win_pdh.UTF16PtrToString(c.SzName)
data[name] = value
}
return data, nil
}
// getQueriedData is used for getting data using the given query handle.
func (p *perfCounterImpl) getQueriedData() ([]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, uint32, error) {
ret := win_pdh.PdhCollectQueryData(p.queryHandle)
if ret != win_pdh.ERROR_SUCCESS {
return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
}
var bufSize, bufCount uint32
var size = uint32(unsafe.Sizeof(win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
var emptyBuf [1]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.
ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &emptyBuf[0])
if ret != win_pdh.PDH_MORE_DATA {
return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
}
filledBuf := make([]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &filledBuf[0])
if ret != win_pdh.ERROR_SUCCESS {
return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
}
return filledBuf, bufCount, nil
}

View File

@ -0,0 +1,86 @@
//go:build windows
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package winstats
import (
"fmt"
"golang.org/x/sys/windows/registry"
)
// OSInfo is a convenience class for retrieving Windows OS information
type OSInfo struct {
BuildNumber, ProductName string
MajorVersion, MinorVersion, UBR uint64
}
// GetOSInfo reads Windows version information from the registry
func GetOSInfo() (*OSInfo, error) {
// for log detail
var keyPrefix string = `regedit:LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion`
k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
if err != nil {
return nil, fmt.Errorf("getOSInfo, open Windows %s failed: %w", keyPrefix, err)
}
defer k.Close()
buildNumber, _, err := k.GetStringValue("CurrentBuildNumber")
if err != nil {
return nil, fmt.Errorf("getOSInfo, get %s\\CurrentBuildNumber failed: %w", keyPrefix, err)
}
majorVersionNumber, _, err := k.GetIntegerValue("CurrentMajorVersionNumber")
if err != nil {
return nil, fmt.Errorf("getOSInfo, get %s\\CurrentMajorVersionNumber failed: %w", keyPrefix, err)
}
minorVersionNumber, _, err := k.GetIntegerValue("CurrentMinorVersionNumber")
if err != nil {
return nil, fmt.Errorf("getOSInfo, get %s\\CurrentMinorVersionNumber failed: %w", keyPrefix, err)
}
revision, _, err := k.GetIntegerValue("UBR")
if err != nil {
return nil, fmt.Errorf("getOSInfo, get %s\\UBR failed: %w", keyPrefix, err)
}
productName, _, err := k.GetStringValue("ProductName")
if err != nil {
return nil, fmt.Errorf("getOSInfo, get %s\\ProductName failed: %w", keyPrefix, err)
}
return &OSInfo{
BuildNumber: buildNumber,
ProductName: productName,
MajorVersion: majorVersionNumber,
MinorVersion: minorVersionNumber,
UBR: revision,
}, nil
}
// GetPatchVersion returns full OS version with patch
func (o *OSInfo) GetPatchVersion() string {
return fmt.Sprintf("%d.%d.%s.%d", o.MajorVersion, o.MinorVersion, o.BuildNumber, o.UBR)
}
// GetBuild returns OS version upto build number
func (o *OSInfo) GetBuild() string {
return fmt.Sprintf("%d.%d.%s", o.MajorVersion, o.MinorVersion, o.BuildNumber)
}

View File

@ -0,0 +1,188 @@
//go:build windows
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package winstats provides a client to get node and pod level stats on windows
package winstats
import (
"syscall"
"time"
"unsafe"
cadvisorapi "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
)
var (
procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
)
// Client is an interface that is used to get stats information.
type Client interface {
WinContainerInfos() (map[string]cadvisorapiv2.ContainerInfo, error)
WinMachineInfo() (*cadvisorapi.MachineInfo, error)
WinVersionInfo() (*cadvisorapi.VersionInfo, error)
GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error)
}
// StatsClient is a client that implements the Client interface
type StatsClient struct {
client winNodeStatsClient
}
type winNodeStatsClient interface {
startMonitoring() error
getNodeMetrics() (nodeMetrics, error)
getNodeInfo() nodeInfo
getMachineInfo() (*cadvisorapi.MachineInfo, error)
getVersionInfo() (*cadvisorapi.VersionInfo, error)
}
type nodeMetrics struct {
cpuUsageCoreNanoSeconds uint64
cpuUsageNanoCores uint64
memoryPrivWorkingSetBytes uint64
memoryCommittedBytes uint64
timeStamp time.Time
interfaceStats []cadvisorapi.InterfaceStats
}
type nodeInfo struct {
memoryPhysicalCapacityBytes uint64
kernelVersion string
osImageVersion string
// startTime is the time when the node was started
startTime time.Time
}
type cpuUsageCoreNanoSecondsCache struct {
latestValue uint64
previousValue uint64
}
// newClient constructs a Client.
func newClient(statsNodeClient winNodeStatsClient) (Client, error) {
statsClient := new(StatsClient)
statsClient.client = statsNodeClient
err := statsClient.client.startMonitoring()
if err != nil {
return nil, err
}
return statsClient, nil
}
// WinContainerInfos returns a map of container infos. The map contains node and
// pod level stats. Analogous to cadvisor GetContainerInfoV2 method.
func (c *StatsClient) WinContainerInfos() (map[string]cadvisorapiv2.ContainerInfo, error) {
infos := make(map[string]cadvisorapiv2.ContainerInfo)
rootContainerInfo, err := c.createRootContainerInfo()
if err != nil {
return nil, err
}
infos["/"] = *rootContainerInfo
return infos, nil
}
// WinMachineInfo returns a cadvisorapi.MachineInfo with details about the
// node machine. Analogous to cadvisor MachineInfo method.
func (c *StatsClient) WinMachineInfo() (*cadvisorapi.MachineInfo, error) {
return c.client.getMachineInfo()
}
// WinVersionInfo returns a cadvisorapi.VersionInfo with version info of
// the kernel and docker runtime. Analogous to cadvisor VersionInfo method.
func (c *StatsClient) WinVersionInfo() (*cadvisorapi.VersionInfo, error) {
return c.client.getVersionInfo()
}
func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, error) {
nodeMetrics, err := c.client.getNodeMetrics()
if err != nil {
return nil, err
}
var stats []*cadvisorapiv2.ContainerStats
stats = append(stats, &cadvisorapiv2.ContainerStats{
Timestamp: nodeMetrics.timeStamp,
Cpu: &cadvisorapi.CpuStats{
Usage: cadvisorapi.CpuUsage{
Total: nodeMetrics.cpuUsageCoreNanoSeconds,
},
},
CpuInst: &cadvisorapiv2.CpuInstStats{
Usage: cadvisorapiv2.CpuInstUsage{
Total: nodeMetrics.cpuUsageNanoCores,
},
},
Memory: &cadvisorapi.MemoryStats{
WorkingSet: nodeMetrics.memoryPrivWorkingSetBytes,
Usage: nodeMetrics.memoryCommittedBytes,
},
Network: &cadvisorapiv2.NetworkStats{
Interfaces: nodeMetrics.interfaceStats,
},
})
nodeInfo := c.client.getNodeInfo()
rootInfo := cadvisorapiv2.ContainerInfo{
Spec: cadvisorapiv2.ContainerSpec{
CreationTime: nodeInfo.startTime,
HasCpu: true,
HasMemory: true,
HasNetwork: true,
Memory: cadvisorapiv2.MemorySpec{
Limit: nodeInfo.memoryPhysicalCapacityBytes,
},
},
Stats: stats,
}
return &rootInfo, nil
}
// GetDirFsInfo returns filesystem capacity and usage information.
func (c *StatsClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
var err error
ret, _, err := syscall.Syscall6(
procGetDiskFreeSpaceEx.Addr(),
4,
uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
uintptr(unsafe.Pointer(&freeBytesAvailable)),
uintptr(unsafe.Pointer(&totalNumberOfBytes)),
uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
0,
0,
)
if ret == 0 {
return cadvisorapiv2.FsInfo{}, err
}
return cadvisorapiv2.FsInfo{
Timestamp: time.Now(),
Capacity: uint64(totalNumberOfBytes),
Available: uint64(freeBytesAvailable),
Usage: uint64(totalNumberOfBytes - freeBytesAvailable),
}, nil
}