rebase: update kubernetes dep to 1.24.0

As kubernetes 1.24.0 is released, updating
kubernetes dependencies to 1.24.0

updates: #3086

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna
2022-05-05 08:17:06 +05:30
committed by mergify[bot]
parent fc1529f268
commit c4f79d455f
959 changed files with 80055 additions and 27456 deletions

View File

@ -0,0 +1,22 @@
// +build linux,go1.16
package selinux
import (
"errors"
"io/fs"
"os"
"github.com/opencontainers/selinux/pkg/pwalkdir"
)
func rchcon(fpath, label string) error {
return pwalkdir.Walk(fpath, func(p string, _ fs.DirEntry, _ error) error {
e := setFileLabel(p, label)
// Walk a file tree can race with removal, so ignore ENOENT.
if errors.Is(e, os.ErrNotExist) {
return nil
}
return e
})
}

View File

@ -0,0 +1,21 @@
// +build linux,!go1.16
package selinux
import (
"errors"
"os"
"github.com/opencontainers/selinux/pkg/pwalk"
)
func rchcon(fpath, label string) error {
return pwalk.Walk(fpath, func(p string, _ os.FileInfo, _ error) error {
e := setFileLabel(p, label)
// Walk a file tree can race with removal, so ignore ENOENT.
if errors.Is(e, os.ErrNotExist) {
return nil
}
return e
})
}

View File

@ -1,7 +1,7 @@
package selinux
import (
"github.com/pkg/errors"
"errors"
)
const (
@ -38,6 +38,8 @@ var (
// CategoryRange allows the upper bound on the category range to be adjusted
CategoryRange = DefaultCategoryRange
privContainerMountLabel string
)
// Context is a representation of the SELinux label broken into 4 parts
@ -59,16 +61,30 @@ func ClassIndex(class string) (int, error) {
return classIndex(class)
}
// SetFileLabel sets the SELinux label for this path or returns an error.
// SetFileLabel sets the SELinux label for this path, following symlinks,
// or returns an error.
func SetFileLabel(fpath string, label string) error {
return setFileLabel(fpath, label)
}
// FileLabel returns the SELinux label for this path or returns an error.
// LsetFileLabel sets the SELinux label for this path, not following symlinks,
// or returns an error.
func LsetFileLabel(fpath string, label string) error {
return lSetFileLabel(fpath, label)
}
// FileLabel returns the SELinux label for this path, following symlinks,
// or returns an error.
func FileLabel(fpath string) (string, error) {
return fileLabel(fpath)
}
// LfileLabel returns the SELinux label for this path, not following symlinks,
// or returns an error.
func LfileLabel(fpath string) (string, error) {
return lFileLabel(fpath)
}
// SetFSCreateLabel tells the kernel what label to use for all file system objects
// created by this task.
// Set the label to an empty string to return to the default label. Calls to SetFSCreateLabel
@ -253,6 +269,8 @@ func CopyLevel(src, dest string) (string, error) {
// Chcon changes the fpath file object to the SELinux label label.
// If fpath is a directory and recurse is true, then Chcon walks the
// directory tree setting the label.
//
// The fpath itself is guaranteed to be relabeled last.
func Chcon(fpath string, label string, recurse bool) error {
return chcon(fpath, label, recurse)
}
@ -280,5 +298,7 @@ func GetDefaultContextWithLevel(user, level, scon string) (string, error) {
// PrivContainerMountLabel returns mount label for privileged containers
func PrivContainerMountLabel() string {
// Make sure label is initialized.
_ = label("")
return privContainerMountLabel
}

View File

@ -5,20 +5,18 @@ import (
"bytes"
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
"math/big"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"github.com/bits-and-blooms/bitset"
"github.com/opencontainers/selinux/pkg/pwalk"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
@ -35,8 +33,6 @@ const (
xattrNameSelinux = "security.selinux"
)
var policyRoot = filepath.Join(selinuxDir, readConfig(selinuxTypeTag))
type selinuxState struct {
enabledSet bool
enabled bool
@ -48,7 +44,7 @@ type selinuxState struct {
type level struct {
sens uint
cats *bitset.BitSet
cats *big.Int
}
type mlsRange struct {
@ -71,7 +67,6 @@ const (
)
var (
assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`)
readOnlyFileLabel string
state = selinuxState{
mcsList: make(map[string]bool),
@ -80,8 +75,24 @@ var (
// for attrPath()
attrPathOnce sync.Once
haveThreadSelf bool
// for policyRoot()
policyRootOnce sync.Once
policyRootVal string
// for label()
loadLabelsOnce sync.Once
labels map[string]string
)
func policyRoot() string {
policyRootOnce.Do(func() {
policyRootVal = filepath.Join(selinuxDir, readConfig(selinuxTypeTag))
})
return policyRootVal
}
func (s *selinuxState) setEnable(enabled bool) bool {
s.Lock()
defer s.Unlock()
@ -120,7 +131,7 @@ func verifySELinuxfsMount(mnt string) bool {
if err == nil {
break
}
if err == unix.EAGAIN || err == unix.EINTR {
if err == unix.EAGAIN || err == unix.EINTR { //nolint:errorlint // unix errors are bare
continue
}
return false
@ -223,7 +234,7 @@ func readConfig(target string) string {
scanner := bufio.NewScanner(in)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
line := bytes.TrimSpace(scanner.Bytes())
if len(line) == 0 {
// Skip blank lines
continue
@ -232,11 +243,12 @@ func readConfig(target string) string {
// Skip comments
continue
}
if groups := assignRegex.FindStringSubmatch(line); groups != nil {
key, val := strings.TrimSpace(groups[1]), strings.TrimSpace(groups[2])
if key == target {
return strings.Trim(val, "\"")
}
fields := bytes.SplitN(line, []byte{'='}, 2)
if len(fields) != 2 {
continue
}
if bytes.Equal(fields[0], []byte(target)) {
return string(bytes.Trim(fields[1], `"`))
}
}
return ""
@ -250,12 +262,12 @@ func isProcHandle(fh *os.File) error {
if err == nil {
break
}
if err != unix.EINTR {
return errors.Wrapf(err, "statfs(%q) failed", fh.Name())
if err != unix.EINTR { //nolint:errorlint // unix errors are bare
return &os.PathError{Op: "fstatfs", Path: fh.Name(), Err: err}
}
}
if buf.Type != unix.PROC_SUPER_MAGIC {
return errors.Errorf("file %q is not on procfs", fh.Name())
return fmt.Errorf("file %q is not on procfs", fh.Name())
}
return nil
@ -275,12 +287,15 @@ func readCon(fpath string) (string, error) {
if err := isProcHandle(in); err != nil {
return "", err
}
return readConFd(in)
}
var retval string
if _, err := fmt.Fscanf(in, "%s", &retval); err != nil {
func readConFd(in *os.File) (string, error) {
data, err := ioutil.ReadAll(in)
if err != nil {
return "", err
}
return strings.Trim(retval, "\x00"), nil
return string(bytes.TrimSuffix(data, []byte{0})), nil
}
// classIndex returns the int index for an object class in the loaded policy,
@ -301,8 +316,9 @@ func classIndex(class string) (int, error) {
return index, nil
}
// setFileLabel sets the SELinux label for this path or returns an error.
func setFileLabel(fpath string, label string) error {
// lSetFileLabel sets the SELinux label for this path, not following symlinks,
// or returns an error.
func lSetFileLabel(fpath string, label string) error {
if fpath == "" {
return ErrEmptyPath
}
@ -311,23 +327,61 @@ func setFileLabel(fpath string, label string) error {
if err == nil {
break
}
if err != unix.EINTR {
return errors.Wrapf(err, "failed to set file label on %s", fpath)
if err != unix.EINTR { //nolint:errorlint // unix errors are bare
return &os.PathError{Op: "lsetxattr", Path: fpath, Err: err}
}
}
return nil
}
// fileLabel returns the SELinux label for this path or returns an error.
// setFileLabel sets the SELinux label for this path, following symlinks,
// or returns an error.
func setFileLabel(fpath string, label string) error {
if fpath == "" {
return ErrEmptyPath
}
for {
err := unix.Setxattr(fpath, xattrNameSelinux, []byte(label), 0)
if err == nil {
break
}
if err != unix.EINTR { //nolint:errorlint // unix errors are bare
return &os.PathError{Op: "setxattr", Path: fpath, Err: err}
}
}
return nil
}
// fileLabel returns the SELinux label for this path, following symlinks,
// or returns an error.
func fileLabel(fpath string) (string, error) {
if fpath == "" {
return "", ErrEmptyPath
}
label, err := getxattr(fpath, xattrNameSelinux)
if err != nil {
return "", &os.PathError{Op: "getxattr", Path: fpath, Err: err}
}
// Trim the NUL byte at the end of the byte buffer, if present.
if len(label) > 0 && label[len(label)-1] == '\x00' {
label = label[:len(label)-1]
}
return string(label), nil
}
// lFileLabel returns the SELinux label for this path, not following symlinks,
// or returns an error.
func lFileLabel(fpath string) (string, error) {
if fpath == "" {
return "", ErrEmptyPath
}
label, err := lgetxattr(fpath, xattrNameSelinux)
if err != nil {
return "", err
return "", &os.PathError{Op: "lgetxattr", Path: fpath, Err: err}
}
// Trim the NUL byte at the end of the byte buffer, if present.
if len(label) > 0 && label[len(label)-1] == '\x00' {
@ -390,7 +444,7 @@ func writeCon(fpath, val string) error {
_, err = out.Write(nil)
}
if err != nil {
return errors.Wrapf(err, "failed to set %s on procfs", fpath)
return err
}
return nil
}
@ -440,8 +494,8 @@ func computeCreateContext(source string, target string, class string) (string, e
}
// catsToBitset stores categories in a bitset.
func catsToBitset(cats string) (*bitset.BitSet, error) {
bitset := &bitset.BitSet{}
func catsToBitset(cats string) (*big.Int, error) {
bitset := new(big.Int)
catlist := strings.Split(cats, ",")
for _, r := range catlist {
@ -456,14 +510,14 @@ func catsToBitset(cats string) (*bitset.BitSet, error) {
return nil, err
}
for i := catstart; i <= catend; i++ {
bitset.Set(i)
bitset.SetBit(bitset, int(i), 1)
}
} else {
cat, err := parseLevelItem(ranges[0], category)
if err != nil {
return nil, err
}
bitset.Set(cat)
bitset.SetBit(bitset, int(cat), 1)
}
}
@ -489,13 +543,13 @@ func (l *level) parseLevel(levelStr string) error {
lvl := strings.SplitN(levelStr, ":", 2)
sens, err := parseLevelItem(lvl[0], sensitivity)
if err != nil {
return errors.Wrap(err, "failed to parse sensitivity")
return fmt.Errorf("failed to parse sensitivity: %w", err)
}
l.sens = sens
if len(lvl) > 1 {
cats, err := catsToBitset(lvl[1])
if err != nil {
return errors.Wrap(err, "failed to parse categories")
return fmt.Errorf("failed to parse categories: %w", err)
}
l.cats = cats
}
@ -513,14 +567,14 @@ func rangeStrToMLSRange(rangeStr string) (*mlsRange, error) {
case 2:
mlsRange.high = &level{}
if err := mlsRange.high.parseLevel(levelSlice[1]); err != nil {
return nil, errors.Wrapf(err, "failed to parse high level %q", levelSlice[1])
return nil, fmt.Errorf("failed to parse high level %q: %w", levelSlice[1], err)
}
fallthrough
// rangeStr that is single level, e.g. s6:c0,c3,c5,c30.c1023
case 1:
mlsRange.low = &level{}
if err := mlsRange.low.parseLevel(levelSlice[0]); err != nil {
return nil, errors.Wrapf(err, "failed to parse low level %q", levelSlice[0])
return nil, fmt.Errorf("failed to parse low level %q: %w", levelSlice[0], err)
}
}
@ -533,37 +587,30 @@ func rangeStrToMLSRange(rangeStr string) (*mlsRange, error) {
// bitsetToStr takes a category bitset and returns it in the
// canonical selinux syntax
func bitsetToStr(c *bitset.BitSet) string {
func bitsetToStr(c *big.Int) string {
var str string
i, e := c.NextSet(0)
len := 0
for e {
if len == 0 {
length := 0
for i := int(c.TrailingZeroBits()); i < c.BitLen(); i++ {
if c.Bit(i) == 0 {
continue
}
if length == 0 {
if str != "" {
str += ","
}
str += "c" + strconv.Itoa(int(i))
str += "c" + strconv.Itoa(i)
}
next, e := c.NextSet(i + 1)
if e {
// consecutive cats
if next == i+1 {
len++
i = next
continue
}
if c.Bit(i+1) == 1 {
length++
continue
}
if len == 1 {
str += ",c" + strconv.Itoa(int(i))
} else if len > 1 {
str += ".c" + strconv.Itoa(int(i))
if length == 1 {
str += ",c" + strconv.Itoa(i)
} else if length > 1 {
str += ".c" + strconv.Itoa(i)
}
if !e {
break
}
len = 0
i = next
length = 0
}
return str
@ -576,13 +623,16 @@ func (l1 *level) equal(l2 *level) bool {
if l1.sens != l2.sens {
return false
}
return l1.cats.Equal(l2.cats)
if l2.cats == nil || l1.cats == nil {
return l2.cats == l1.cats
}
return l1.cats.Cmp(l2.cats) == 0
}
// String returns an mlsRange as a string.
func (m mlsRange) String() string {
low := "s" + strconv.Itoa(int(m.low.sens))
if m.low.cats != nil && m.low.cats.Count() > 0 {
if m.low.cats != nil && m.low.cats.BitLen() > 0 {
low += ":" + bitsetToStr(m.low.cats)
}
@ -591,7 +641,7 @@ func (m mlsRange) String() string {
}
high := "s" + strconv.Itoa(int(m.high.sens))
if m.high.cats != nil && m.high.cats.Count() > 0 {
if m.high.cats != nil && m.high.cats.BitLen() > 0 {
high += ":" + bitsetToStr(m.high.cats)
}
@ -641,10 +691,12 @@ func calculateGlbLub(sourceRange, targetRange string) (string, error) {
/* find the intersecting categories */
if s.low.cats != nil && t.low.cats != nil {
outrange.low.cats = s.low.cats.Intersection(t.low.cats)
outrange.low.cats = new(big.Int)
outrange.low.cats.And(s.low.cats, t.low.cats)
}
if s.high.cats != nil && t.high.cats != nil {
outrange.high.cats = s.high.cats.Intersection(t.high.cats)
outrange.high.cats = new(big.Int)
outrange.high.cats.And(s.high.cats, t.high.cats)
}
return outrange.String(), nil
@ -665,11 +717,7 @@ func readWriteCon(fpath string, val string) (string, error) {
return "", err
}
var retval string
if _, err := fmt.Fscanf(f, "%s", &retval); err != nil {
return "", err
}
return strings.Trim(retval, "\x00"), nil
return readConFd(f)
}
// setExecLabel sets the SELinux label that the kernel will use for any programs
@ -697,17 +745,21 @@ func socketLabel() (string, error) {
// peerLabel retrieves the label of the client on the other side of a socket
func peerLabel(fd uintptr) (string, error) {
return unix.GetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_PEERSEC)
label, err := unix.GetsockoptString(int(fd), unix.SOL_SOCKET, unix.SO_PEERSEC)
if err != nil {
return "", &os.PathError{Op: "getsockopt", Path: "fd " + strconv.Itoa(int(fd)), Err: err}
}
return label, nil
}
// setKeyLabel takes a process label and tells the kernel to assign the
// label to the next kernel keyring that gets created
func setKeyLabel(label string) error {
err := writeCon("/proc/self/attr/keycreate", label)
if os.IsNotExist(errors.Cause(err)) {
if errors.Is(err, os.ErrNotExist) {
return nil
}
if label == "" && os.IsPermission(errors.Cause(err)) {
if label == "" && errors.Is(err, os.ErrPermission) {
return nil
}
return err
@ -720,10 +772,10 @@ func keyLabel() (string, error) {
// get returns the Context as a string
func (c Context) get() string {
if c["level"] != "" {
return fmt.Sprintf("%s:%s:%s:%s", c["user"], c["role"], c["type"], c["level"])
if level := c["level"]; level != "" {
return c["user"] + ":" + c["role"] + ":" + c["type"] + ":" + level
}
return fmt.Sprintf("%s:%s:%s", c["user"], c["role"], c["type"])
return c["user"] + ":" + c["role"] + ":" + c["type"]
}
// newContext creates a new Context struct from the specified label
@ -784,7 +836,7 @@ func enforceMode() int {
// setEnforceMode sets the current SELinux mode Enforcing, Permissive.
// Disabled is not valid, since this needs to be set at boot time.
func setEnforceMode(mode int) error {
return ioutil.WriteFile(selinuxEnforcePath(), []byte(strconv.Itoa(mode)), 0644)
return ioutil.WriteFile(selinuxEnforcePath(), []byte(strconv.Itoa(mode)), 0o644)
}
// defaultEnforceMode returns the systems default SELinux mode Enforcing,
@ -888,24 +940,21 @@ func openContextFile() (*os.File, error) {
if f, err := os.Open(contextFile); err == nil {
return f, nil
}
lxcPath := filepath.Join(policyRoot, "/contexts/lxc_contexts")
return os.Open(lxcPath)
return os.Open(filepath.Join(policyRoot(), "/contexts/lxc_contexts"))
}
var labels, privContainerMountLabel = loadLabels()
func loadLabels() (map[string]string, string) {
labels := make(map[string]string)
func loadLabels() {
labels = make(map[string]string)
in, err := openContextFile()
if err != nil {
return labels, ""
return
}
defer in.Close()
scanner := bufio.NewScanner(in)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
line := bytes.TrimSpace(scanner.Bytes())
if len(line) == 0 {
// Skip blank lines
continue
@ -914,38 +963,47 @@ func loadLabels() (map[string]string, string) {
// Skip comments
continue
}
if groups := assignRegex.FindStringSubmatch(line); groups != nil {
key, val := strings.TrimSpace(groups[1]), strings.TrimSpace(groups[2])
labels[key] = strings.Trim(val, "\"")
fields := bytes.SplitN(line, []byte{'='}, 2)
if len(fields) != 2 {
continue
}
key, val := bytes.TrimSpace(fields[0]), bytes.TrimSpace(fields[1])
labels[string(key)] = string(bytes.Trim(val, `"`))
}
con, _ := NewContext(labels["file"])
con["level"] = fmt.Sprintf("s0:c%d,c%d", maxCategory-2, maxCategory-1)
reserveLabel(con.get())
return labels, con.get()
privContainerMountLabel = con.get()
reserveLabel(privContainerMountLabel)
}
func label(key string) string {
loadLabelsOnce.Do(func() {
loadLabels()
})
return labels[key]
}
// kvmContainerLabels returns the default processLabel and mountLabel to be used
// for kvm containers by the calling process.
func kvmContainerLabels() (string, string) {
processLabel := labels["kvm_process"]
processLabel := label("kvm_process")
if processLabel == "" {
processLabel = labels["process"]
processLabel = label("process")
}
return addMcs(processLabel, labels["file"])
return addMcs(processLabel, label("file"))
}
// initContainerLabels returns the default processLabel and file labels to be
// used for containers running an init system like systemd by the calling process.
func initContainerLabels() (string, string) {
processLabel := labels["init_process"]
processLabel := label("init_process")
if processLabel == "" {
processLabel = labels["process"]
processLabel = label("process")
}
return addMcs(processLabel, labels["file"])
return addMcs(processLabel, label("file"))
}
// containerLabels returns an allocated processLabel and fileLabel to be used for
@ -955,9 +1013,9 @@ func containerLabels() (processLabel string, fileLabel string) {
return "", ""
}
processLabel = labels["process"]
fileLabel = labels["file"]
readOnlyFileLabel = labels["ro_file"]
processLabel = label("process")
fileLabel = label("file")
readOnlyFileLabel = label("ro_file")
if processLabel == "" || fileLabel == "" {
return "", fileLabel
@ -985,7 +1043,7 @@ func addMcs(processLabel, fileLabel string) (string, string) {
// securityCheckContext validates that the SELinux label is understood by the kernel
func securityCheckContext(val string) error {
return ioutil.WriteFile(path.Join(getSelinuxMountPoint(), "context"), []byte(val), 0644)
return ioutil.WriteFile(path.Join(getSelinuxMountPoint(), "context"), []byte(val), 0o644)
}
// copyLevel returns a label with the MLS/MCS level from src label replaced on
@ -1023,7 +1081,7 @@ func badPrefix(fpath string) error {
badPrefixes := []string{"/usr"}
for _, prefix := range badPrefixes {
if strings.HasPrefix(fpath, prefix) {
return errors.Errorf("relabeling content in %s is not allowed", prefix)
return fmt.Errorf("relabeling content in %s is not allowed", prefix)
}
}
return nil
@ -1044,17 +1102,10 @@ func chcon(fpath string, label string, recurse bool) error {
}
if !recurse {
return SetFileLabel(fpath, label)
return setFileLabel(fpath, label)
}
return pwalk.Walk(fpath, func(p string, info os.FileInfo, err error) error {
e := SetFileLabel(p, label)
// Walk a file tree can race with removal, so ignore ENOENT
if os.IsNotExist(errors.Cause(e)) {
return nil
}
return e
})
return rchcon(fpath, label)
}
// dupSecOpt takes an SELinux process label and returns security options that
@ -1072,7 +1123,8 @@ func dupSecOpt(src string) ([]string, error) {
con["type"] == "" {
return nil, nil
}
dup := []string{"user:" + con["user"],
dup := []string{
"user:" + con["user"],
"role:" + con["role"],
"type:" + con["type"],
}
@ -1140,9 +1192,8 @@ func findUserInContext(context Context, r io.Reader, verifier func(string) error
return outConn, nil
}
}
if err := scanner.Err(); err != nil {
return "", errors.Wrap(err, "failed to scan for context")
return "", fmt.Errorf("failed to scan for context: %w", err)
}
return "", nil
@ -1155,7 +1206,7 @@ func getDefaultContextFromReaders(c *defaultSECtx) (string, error) {
context, err := newContext(c.scon)
if err != nil {
return "", errors.Wrapf(err, "failed to create label for %s", c.scon)
return "", fmt.Errorf("failed to create label for %s: %w", c.scon, err)
}
// set so the verifier validates the matched context with the provided user and level.
@ -1180,19 +1231,18 @@ func getDefaultContextFromReaders(c *defaultSECtx) (string, error) {
return conn, nil
}
return "", errors.Wrapf(ErrContextMissing, "context not found: %q", c.scon)
return "", fmt.Errorf("context %q not found: %w", c.scon, ErrContextMissing)
}
func getDefaultContextWithLevel(user, level, scon string) (string, error) {
userPath := filepath.Join(policyRoot, selinuxUsersDir, user)
defaultPath := filepath.Join(policyRoot, defaultContexts)
userPath := filepath.Join(policyRoot(), selinuxUsersDir, user)
fu, err := os.Open(userPath)
if err != nil {
return "", err
}
defer fu.Close()
defaultPath := filepath.Join(policyRoot(), defaultContexts)
fd, err := os.Open(defaultPath)
if err != nil {
return "", err

View File

@ -2,8 +2,6 @@
package selinux
const privContainerMountLabel = ""
func setDisabled() {
}
@ -19,10 +17,18 @@ func setFileLabel(fpath string, label string) error {
return nil
}
func lSetFileLabel(fpath string, label string) error {
return nil
}
func fileLabel(fpath string) (string, error) {
return "", nil
}
func lFileLabel(fpath string) (string, error) {
return "", nil
}
func setFSCreateLabel(label string) error {
return nil
}
@ -152,3 +158,7 @@ func disableSecOpt() []string {
func getDefaultContextWithLevel(user, level, scon string) (string, error) {
return "", nil
}
func label(_ string) string {
return ""
}

View File

@ -10,7 +10,7 @@ func lgetxattr(path, attr string) ([]byte, error) {
// Start with a 128 length byte array
dest := make([]byte, 128)
sz, errno := doLgetxattr(path, attr, dest)
for errno == unix.ERANGE {
for errno == unix.ERANGE { //nolint:errorlint // unix errors are bare
// Buffer too small, use zero-sized buffer to get the actual size
sz, errno = doLgetxattr(path, attr, []byte{})
if errno != nil {
@ -31,7 +31,40 @@ func lgetxattr(path, attr string) ([]byte, error) {
func doLgetxattr(path, attr string, dest []byte) (int, error) {
for {
sz, err := unix.Lgetxattr(path, attr, dest)
if err != unix.EINTR {
if err != unix.EINTR { //nolint:errorlint // unix errors are bare
return sz, err
}
}
}
// getxattr returns a []byte slice containing the value of
// an extended attribute attr set for path.
func getxattr(path, attr string) ([]byte, error) {
// Start with a 128 length byte array
dest := make([]byte, 128)
sz, errno := dogetxattr(path, attr, dest)
for errno == unix.ERANGE { //nolint:errorlint // unix errors are bare
// Buffer too small, use zero-sized buffer to get the actual size
sz, errno = dogetxattr(path, attr, []byte{})
if errno != nil {
return nil, errno
}
dest = make([]byte, sz)
sz, errno = dogetxattr(path, attr, dest)
}
if errno != nil {
return nil, errno
}
return dest[:sz], nil
}
// dogetxattr is a wrapper that retries on EINTR
func dogetxattr(path, attr string, dest []byte) (int, error) {
for {
sz, err := unix.Getxattr(path, attr, dest)
if err != unix.EINTR { //nolint:errorlint // unix errors are bare
return sz, err
}
}

View File

@ -8,6 +8,12 @@ By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks.
This can be changed by using WalkN function which has the additional
parameter, specifying the number of goroutines (concurrency).
### pwalk vs pwalkdir
This package is deprecated in favor of
[pwalkdir](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir),
which is faster, but requires at least Go 1.16.
### Caveats
Please note the following limitations of this code:

View File

@ -1,12 +1,11 @@
package pwalk
import (
"fmt"
"os"
"path/filepath"
"runtime"
"sync"
"github.com/pkg/errors"
)
type WalkFunc = filepath.WalkFunc
@ -20,7 +19,7 @@ type WalkFunc = filepath.WalkFunc
//
// Note that this implementation only supports primitive error handling:
//
// - no errors are ever passed to WalkFn;
// - no errors are ever passed to walkFn;
//
// - once a walkFn returns any error, all further processing stops
// and the error is returned to the caller of Walk;
@ -42,7 +41,7 @@ func Walk(root string, walkFn WalkFunc) error {
func WalkN(root string, walkFn WalkFunc, num int) error {
// make sure limit is sensible
if num < 1 {
return errors.Errorf("walk(%q): num must be > 0", root)
return fmt.Errorf("walk(%q): num must be > 0", root)
}
files := make(chan *walkArgs, 2*num)
@ -52,6 +51,9 @@ func WalkN(root string, walkFn WalkFunc, num int) error {
var (
err error
wg sync.WaitGroup
rootLen = len(root)
rootEntry *walkArgs
)
wg.Add(1)
go func() {
@ -60,6 +62,11 @@ func WalkN(root string, walkFn WalkFunc, num int) error {
close(files)
return err
}
if len(p) == rootLen {
// Root entry is processed separately below.
rootEntry = &walkArgs{path: p, info: &info}
return nil
}
// add a file to the queue unless a callback sent an error
select {
case e := <-errCh:
@ -93,10 +100,14 @@ func WalkN(root string, walkFn WalkFunc, num int) error {
wg.Wait()
if err == nil {
err = walkFn(rootEntry.path, *rootEntry.info, nil)
}
return err
}
// walkArgs holds the arguments that were passed to the Walk or WalkLimit
// walkArgs holds the arguments that were passed to the Walk or WalkN
// functions.
type walkArgs struct {
path string

View File

@ -0,0 +1,54 @@
## pwalkdir: parallel implementation of filepath.WalkDir
This is a wrapper for [filepath.WalkDir](https://pkg.go.dev/path/filepath#WalkDir)
which may speed it up by calling multiple callback functions (WalkDirFunc)
in parallel, utilizing goroutines.
By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks.
This can be changed by using WalkN function which has the additional
parameter, specifying the number of goroutines (concurrency).
### pwalk vs pwalkdir
This package is very similar to
[pwalk](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir),
but utilizes `filepath.WalkDir` (added to Go 1.16), which does not call stat(2)
on every entry and is therefore faster (up to 3x, depending on usage scenario).
Users who are OK with requiring Go 1.16+ should switch to this
implementation.
### Caveats
Please note the following limitations of this code:
* Unlike filepath.WalkDir, the order of calls is non-deterministic;
* Only primitive error handling is supported:
* fs.SkipDir is not supported;
* no errors are ever passed to WalkDirFunc;
* once any error is returned from any walkDirFunc instance, no more calls
to WalkDirFunc are made, and the error is returned to the caller of WalkDir;
* if more than one WalkDirFunc instance will return an error, only one
of such errors will be propagated to and returned by WalkDir, others
will be silently discarded.
### Documentation
For the official documentation, see
https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir
### Benchmarks
For a WalkDirFunc that consists solely of the return statement, this
implementation is about 15% slower than the standard library's
filepath.WalkDir.
Otherwise (if a WalkDirFunc is actually doing something) this is usually
faster, except when the WalkDirN(..., 1) is used. Run `go test -bench .`
to see how different operations can benefit from it, as well as how the
level of paralellism affects the speed.

View File

@ -0,0 +1,116 @@
//go:build go1.16
// +build go1.16
package pwalkdir
import (
"fmt"
"io/fs"
"path/filepath"
"runtime"
"sync"
)
// Walk is a wrapper for filepath.WalkDir which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// twice the runtime.NumCPU() walkFn will be called at any one time.
// If you want to change the maximum, use WalkN instead.
//
// The order of calls is non-deterministic.
//
// Note that this implementation only supports primitive error handling:
//
// - no errors are ever passed to walkFn;
//
// - once a walkFn returns any error, all further processing stops
// and the error is returned to the caller of Walk;
//
// - filepath.SkipDir is not supported;
//
// - if more than one walkFn instance will return an error, only one
// of such errors will be propagated and returned by Walk, others
// will be silently discarded.
func Walk(root string, walkFn fs.WalkDirFunc) error {
return WalkN(root, walkFn, runtime.NumCPU()*2)
}
// WalkN is a wrapper for filepath.WalkDir which can call multiple walkFn
// in parallel, allowing to handle each item concurrently. A maximum of
// num walkFn will be called at any one time.
//
// Please see Walk documentation for caveats of using this function.
func WalkN(root string, walkFn fs.WalkDirFunc, num int) error {
// make sure limit is sensible
if num < 1 {
return fmt.Errorf("walk(%q): num must be > 0", root)
}
files := make(chan *walkArgs, 2*num)
errCh := make(chan error, 1) // Get the first error, ignore others.
// Start walking a tree asap.
var (
err error
wg sync.WaitGroup
rootLen = len(root)
rootEntry *walkArgs
)
wg.Add(1)
go func() {
err = filepath.WalkDir(root, func(p string, entry fs.DirEntry, err error) error {
if err != nil {
close(files)
return err
}
if len(p) == rootLen {
// Root entry is processed separately below.
rootEntry = &walkArgs{path: p, entry: entry}
return nil
}
// Add a file to the queue unless a callback sent an error.
select {
case e := <-errCh:
close(files)
return e
default:
files <- &walkArgs{path: p, entry: entry}
return nil
}
})
if err == nil {
close(files)
}
wg.Done()
}()
wg.Add(num)
for i := 0; i < num; i++ {
go func() {
for file := range files {
if e := walkFn(file.path, file.entry, nil); e != nil {
select {
case errCh <- e: // sent ok
default: // buffer full
}
}
}
wg.Done()
}()
}
wg.Wait()
if err == nil {
err = walkFn(rootEntry.path, rootEntry.entry, nil)
}
return err
}
// walkArgs holds the arguments that were passed to the Walk or WalkN
// functions.
type walkArgs struct {
path string
entry fs.DirEntry
}