build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>
This commit is contained in:
Niels de Vos
2025-03-04 08:57:28 +01:00
committed by mergify[bot]
parent 15da101b1b
commit bec6090996
8047 changed files with 1407827 additions and 3453 deletions

View File

@ -0,0 +1,172 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"time"
)
// DefaultFs implements Filesystem using same-named functions from "os" and "io"
type DefaultFs struct {
root string
}
var _ Filesystem = &DefaultFs{}
// NewTempFs returns a fake Filesystem in temporary directory, useful for unit tests
func NewTempFs() Filesystem {
path, _ := os.MkdirTemp("", "tmpfs")
return &DefaultFs{
root: path,
}
}
func (fs *DefaultFs) prefix(path string) string {
if len(fs.root) == 0 {
return path
}
return filepath.Join(fs.root, path)
}
// Stat via os.Stat
func (fs *DefaultFs) Stat(name string) (os.FileInfo, error) {
return os.Stat(fs.prefix(name))
}
// Create via os.Create
func (fs *DefaultFs) Create(name string) (File, error) {
file, err := os.Create(fs.prefix(name))
if err != nil {
return nil, err
}
return &defaultFile{file}, nil
}
// Rename via os.Rename
func (fs *DefaultFs) Rename(oldpath, newpath string) error {
if !strings.HasPrefix(oldpath, fs.root) {
oldpath = fs.prefix(oldpath)
}
if !strings.HasPrefix(newpath, fs.root) {
newpath = fs.prefix(newpath)
}
return os.Rename(oldpath, newpath)
}
func (fs *DefaultFs) MkdirAll(path string, perm os.FileMode) error {
return MkdirAll(fs.prefix(path), perm)
}
// MkdirAllWithPathCheck checks if path exists already. If not, it creates a directory
// named path, along with any necessary parents, and returns nil, or else returns an error.
// Permission bits perm (before umask) are used for all directories that
// MkdirAllWithPathCheck creates.
// If path is already a directory, MkdirAllWithPathCheck does nothing and returns nil.
// NOTE: In case of Windows NTFS, mount points are implemented as reparse-point
// (similar to symlink) and do not represent actual directory. Hence Directory existence
// check for windows NTFS will NOT check for dir, but for symlink presence.
func MkdirAllWithPathCheck(path string, perm os.FileMode) error {
if dir, err := os.Lstat(path); err == nil {
// If the path exists already,
// 1. for Unix/Linux OS, check if the path is directory.
// 2. for windows NTFS, check if the path is symlink instead of directory.
if dir.IsDir() ||
(runtime.GOOS == "windows" && (dir.Mode()&os.ModeSymlink != 0)) {
return nil
}
return fmt.Errorf("path %v exists but is not a directory", path)
}
// If existence of path not known, attempt to create it.
if err := MkdirAll(path, perm); err != nil {
return err
}
return nil
}
// Chtimes via os.Chtimes
func (fs *DefaultFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(fs.prefix(name), atime, mtime)
}
// RemoveAll via os.RemoveAll
func (fs *DefaultFs) RemoveAll(path string) error {
return os.RemoveAll(fs.prefix(path))
}
// Remove via os.Remove
func (fs *DefaultFs) Remove(name string) error {
return os.Remove(fs.prefix(name))
}
// ReadFile via os.ReadFile
func (fs *DefaultFs) ReadFile(filename string) ([]byte, error) {
return os.ReadFile(fs.prefix(filename))
}
// TempDir via os.MkdirTemp
func (fs *DefaultFs) TempDir(dir, prefix string) (string, error) {
return os.MkdirTemp(fs.prefix(dir), prefix)
}
// TempFile via os.CreateTemp
func (fs *DefaultFs) TempFile(dir, prefix string) (File, error) {
file, err := os.CreateTemp(fs.prefix(dir), prefix)
if err != nil {
return nil, err
}
return &defaultFile{file}, nil
}
// ReadDir via os.ReadDir
func (fs *DefaultFs) ReadDir(dirname string) ([]os.DirEntry, error) {
return os.ReadDir(fs.prefix(dirname))
}
// Walk via filepath.Walk
func (fs *DefaultFs) Walk(root string, walkFn filepath.WalkFunc) error {
return filepath.Walk(fs.prefix(root), walkFn)
}
// defaultFile implements File using same-named functions from "os"
type defaultFile struct {
file *os.File
}
// Name via os.File.Name
func (file *defaultFile) Name() string {
return file.file.Name()
}
// Write via os.File.Write
func (file *defaultFile) Write(b []byte) (n int, err error) {
return file.file.Write(b)
}
// Sync via os.File.Sync
func (file *defaultFile) Sync() error {
return file.file.Sync()
}
// Close via os.File.Close
func (file *defaultFile) Close() error {
return file.file.Close()
}

View File

@ -0,0 +1,52 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"os"
"path/filepath"
"time"
)
// Filesystem is an interface that we can use to mock various filesystem operations
type Filesystem interface {
// from "os"
Stat(name string) (os.FileInfo, error)
Create(name string) (File, error)
Rename(oldpath, newpath string) error
MkdirAll(path string, perm os.FileMode) error
Chtimes(name string, atime time.Time, mtime time.Time) error
RemoveAll(path string) error
Remove(name string) error
// from "os"
ReadFile(filename string) ([]byte, error)
TempDir(dir, prefix string) (string, error)
TempFile(dir, prefix string) (File, error)
ReadDir(dirname string) ([]os.DirEntry, error)
Walk(root string, walkFn filepath.WalkFunc) error
}
// File is an interface that we can use to mock various filesystem operations typically
// accessed through the File object from the "os" package
type File interface {
// for now, the only os.File methods used are those below, add more as necessary
Name() string
Write(b []byte) (n int, err error)
Sync() error
Close() error
}

View File

@ -0,0 +1,27 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"path/filepath"
)
// IsPathClean will replace slashes to Separator (which is OS-specific).
// This will make sure that all slashes are the same before comparing.
func IsPathClean(path string) bool {
return filepath.ToSlash(filepath.Clean(path)) == filepath.ToSlash(path)
}

View File

@ -0,0 +1,53 @@
//go:build freebsd || linux || darwin
// +build freebsd linux darwin
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"fmt"
"os"
"path/filepath"
)
// IsUnixDomainSocket returns whether a given file is a AF_UNIX socket file
func IsUnixDomainSocket(filePath string) (bool, error) {
fi, err := os.Stat(filePath)
if err != nil {
return false, fmt.Errorf("stat file %s failed: %v", filePath, err)
}
if fi.Mode()&os.ModeSocket == 0 {
return false, nil
}
return true, nil
}
// Chmod is the same as os.Chmod on Unix.
func Chmod(name string, mode os.FileMode) error {
return os.Chmod(name, mode)
}
// MkdirAll is same as os.MkdirAll on Unix.
func MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
// IsAbs is same as filepath.IsAbs on Unix.
func IsAbs(path string) bool {
return filepath.IsAbs(path)
}

View File

@ -0,0 +1,255 @@
//go:build windows
// +build windows
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
"time"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
"golang.org/x/sys/windows"
)
const (
// Amount of time to wait between attempting to use a Unix domain socket.
// As detailed in https://github.com/kubernetes/kubernetes/issues/104584
// the first attempt will most likely fail, hence the need to retry
socketDialRetryPeriod = 1 * time.Second
// Overall timeout value to dial a Unix domain socket, including retries
socketDialTimeout = 4 * time.Second
)
// IsUnixDomainSocket returns whether a given file is a AF_UNIX socket file
// Note that due to the retry logic inside, it could take up to 4 seconds
// to determine whether or not the file path supplied is a Unix domain socket
func IsUnixDomainSocket(filePath string) (bool, error) {
// Due to the absence of golang support for os.ModeSocket in Windows (https://github.com/golang/go/issues/33357)
// we need to dial the file and check if we receive an error to determine if a file is Unix Domain Socket file.
// Note that querrying for the Reparse Points (https://docs.microsoft.com/en-us/windows/win32/fileio/reparse-points)
// for the file (using FSCTL_GET_REPARSE_POINT) and checking for reparse tag: reparseTagSocket
// does NOT work in 1809 if the socket file is created within a bind mounted directory by a container
// and the FSCTL is issued in the host by the kubelet.
// If the file does not exist, it cannot be a Unix domain socket.
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return false, fmt.Errorf("File %s not found. Err: %v", filePath, err)
}
klog.V(6).InfoS("Function IsUnixDomainSocket starts", "filePath", filePath)
// As detailed in https://github.com/kubernetes/kubernetes/issues/104584 we cannot rely
// on the Unix Domain socket working on the very first try, hence the potential need to
// dial multiple times
var lastSocketErr error
err := wait.PollImmediate(socketDialRetryPeriod, socketDialTimeout,
func() (bool, error) {
klog.V(6).InfoS("Dialing the socket", "filePath", filePath)
var c net.Conn
c, lastSocketErr = net.Dial("unix", filePath)
if lastSocketErr == nil {
c.Close()
klog.V(6).InfoS("Socket dialed successfully", "filePath", filePath)
return true, nil
}
klog.V(6).InfoS("Failed the current attempt to dial the socket, so pausing before retry",
"filePath", filePath, "err", lastSocketErr, "socketDialRetryPeriod",
socketDialRetryPeriod)
return false, nil
})
// PollImmediate will return "timed out waiting for the condition" if the function it
// invokes never returns true
if err != nil {
klog.V(2).InfoS("Failed all attempts to dial the socket so marking it as a non-Unix Domain socket. Last socket error along with the error from PollImmediate follow",
"filePath", filePath, "lastSocketErr", lastSocketErr, "err", err)
return false, nil
}
return true, nil
}
// On Windows os.Mkdir all doesn't set any permissions so call the Chown function below to set
// permissions once the directory is created.
func MkdirAll(path string, perm os.FileMode) error {
klog.V(6).InfoS("Function MkdirAll starts", "path", path, "perm", perm)
err := os.MkdirAll(path, perm)
if err != nil {
return fmt.Errorf("Error creating directory %s: %v", path, err)
}
err = Chmod(path, perm)
if err != nil {
return fmt.Errorf("Error setting permissions for directory %s: %v", path, err)
}
return nil
}
const (
// These aren't defined in the syscall package for Windows :(
USER_READ = 0x100
USER_WRITE = 0x80
USER_EXECUTE = 0x40
GROUP_READ = 0x20
GROUP_WRITE = 0x10
GROUP_EXECUTE = 0x8
OTHERS_READ = 0x4
OTHERS_WRITE = 0x2
OTHERS_EXECUTE = 0x1
USER_ALL = USER_READ | USER_WRITE | USER_EXECUTE
GROUP_ALL = GROUP_READ | GROUP_WRITE | GROUP_EXECUTE
OTHERS_ALL = OTHERS_READ | OTHERS_WRITE | OTHERS_EXECUTE
)
// On Windows os.Chmod only sets the read-only flag on files, so we need to use Windows APIs to set the desired access on files / directories.
// The OWNER mode will set file permissions for the file owner SID, the GROUP mode will set file permissions for the file group SID,
// and the OTHERS mode will set file permissions for BUILTIN\Users.
// Please note that Windows containers can be run as one of two user accounts; ContainerUser or ContainerAdministrator.
// Containers run as ContainerAdministrator will inherit permissions from BUILTIN\Administrators,
// while containers run as ContainerUser will inherit permissions from BUILTIN\Users.
// Windows containers do not have the ability to run as a custom user account that is known to the host so the OTHERS group mode
// is used to grant / deny permissions of files on the hosts to the ContainerUser account.
func Chmod(path string, filemode os.FileMode) error {
klog.V(6).InfoS("Function Chmod starts", "path", path, "filemode", filemode)
// Get security descriptor for the file
sd, err := windows.GetNamedSecurityInfo(
path,
windows.SE_FILE_OBJECT,
windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION|windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION)
if err != nil {
return fmt.Errorf("Error getting security descriptor for file %s: %v", path, err)
}
// Get owner SID from the security descriptor for assigning USER permissions
owner, _, err := sd.Owner()
if err != nil {
return fmt.Errorf("Error getting owner SID for file %s: %v", path, err)
}
ownerString := owner.String()
// Get the group SID from the security descriptor for assigning GROUP permissions
group, _, err := sd.Group()
if err != nil {
return fmt.Errorf("Error getting group SID for file %s: %v", path, err)
}
groupString := group.String()
mask := uint32(windows.ACCESS_MASK(filemode))
// Build a new Discretionary Access Control List (DACL) with the desired permissions using
//the Security Descriptor Definition Language (SDDL) format.
// https://learn.microsoft.com/windows/win32/secauthz/security-descriptor-definition-language
// the DACL is a list of Access Control Entries (ACEs) where each ACE represents the permissions (Allow or Deny) for a specific SID.
// Each ACE has the following format:
// (AceType;AceFlags;Rights;ObjectGuid;InheritObjectGuid;AccountSid)
// We can leave ObjectGuid and InheritObjectGuid empty for our purposes.
dacl := "D:"
// build the owner ACE
dacl += "(A;OICI;"
if mask&USER_ALL == USER_ALL {
dacl += "FA"
} else {
if mask&USER_READ == USER_READ {
dacl += "FR"
}
if mask&USER_WRITE == USER_WRITE {
dacl += "FW"
}
if mask&USER_EXECUTE == USER_EXECUTE {
dacl += "FX"
}
}
dacl += ";;;" + ownerString + ")"
// Build the group ACE
dacl += "(A;OICI;"
if mask&GROUP_ALL == GROUP_ALL {
dacl += "FA"
} else {
if mask&GROUP_READ == GROUP_READ {
dacl += "FR"
}
if mask&GROUP_WRITE == GROUP_WRITE {
dacl += "FW"
}
if mask&GROUP_EXECUTE == GROUP_EXECUTE {
dacl += "FX"
}
}
dacl += ";;;" + groupString + ")"
// Build the others ACE
dacl += "(A;OICI;"
if mask&OTHERS_ALL == OTHERS_ALL {
dacl += "FA"
} else {
if mask&OTHERS_READ == OTHERS_READ {
dacl += "FR"
}
if mask&OTHERS_WRITE == OTHERS_WRITE {
dacl += "FW"
}
if mask&OTHERS_EXECUTE == OTHERS_EXECUTE {
dacl += "FX"
}
}
dacl += ";;;BU)"
klog.V(6).InfoS("Setting new DACL for path", "path", path, "dacl", dacl)
// create a new security descriptor from the DACL string
newSD, err := windows.SecurityDescriptorFromString(dacl)
if err != nil {
return fmt.Errorf("Error creating new security descriptor from DACL string: %v", err)
}
// get the DACL in binary format from the newly created security descriptor
newDACL, _, err := newSD.DACL()
if err != nil {
return fmt.Errorf("Error getting DACL from new security descriptor: %v", err)
}
// Write the new security descriptor to the file
return windows.SetNamedSecurityInfo(
path,
windows.SE_FILE_OBJECT,
windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION,
nil, // owner SID
nil, // group SID
newDACL,
nil) // SACL
}
// IsAbs returns whether the given path is absolute or not.
// On Windows, filepath.IsAbs will not return True for paths prefixed with a slash, even
// though they can be used as absolute paths (https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats).
//
// WARN: It isn't safe to use this for API values which will propagate across systems (e.g. REST API values
// that get validated on Unix, persisted, then consumed by Windows, etc).
func IsAbs(path string) bool {
return filepath.IsAbs(path) || strings.HasPrefix(path, `\`) || strings.HasPrefix(path, `/`)
}

View File

@ -0,0 +1,216 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filesystem
import (
"context"
"fmt"
"time"
"github.com/fsnotify/fsnotify"
)
// FSWatcher is a callback-based filesystem watcher abstraction for fsnotify.
type FSWatcher interface {
// Initializes the watcher with the given watch handlers.
// Called before all other methods.
Init(FSEventHandler, FSErrorHandler) error
// Starts listening for events and errors.
// When an event or error occurs, the corresponding handler is called.
Run()
// Add a filesystem path to watch
AddWatch(path string) error
}
// FSEventHandler is called when a fsnotify event occurs.
type FSEventHandler func(event fsnotify.Event)
// FSErrorHandler is called when a fsnotify error occurs.
type FSErrorHandler func(err error)
type fsnotifyWatcher struct {
watcher *fsnotify.Watcher
eventHandler FSEventHandler
errorHandler FSErrorHandler
}
var _ FSWatcher = &fsnotifyWatcher{}
// NewFsnotifyWatcher returns an implementation of FSWatcher that continuously listens for
// fsnotify events and calls the event handler as soon as an event is received.
func NewFsnotifyWatcher() FSWatcher {
return &fsnotifyWatcher{}
}
func (w *fsnotifyWatcher) AddWatch(path string) error {
return w.watcher.Add(path)
}
func (w *fsnotifyWatcher) Init(eventHandler FSEventHandler, errorHandler FSErrorHandler) error {
var err error
w.watcher, err = fsnotify.NewWatcher()
if err != nil {
return err
}
w.eventHandler = eventHandler
w.errorHandler = errorHandler
return nil
}
func (w *fsnotifyWatcher) Run() {
go func() {
defer w.watcher.Close()
for {
select {
case event := <-w.watcher.Events:
if w.eventHandler != nil {
w.eventHandler(event)
}
case err := <-w.watcher.Errors:
if w.errorHandler != nil {
w.errorHandler(err)
}
}
}
}()
}
type watchAddRemover interface {
Add(path string) error
Remove(path string) error
}
type noopWatcher struct{}
func (noopWatcher) Add(path string) error { return nil }
func (noopWatcher) Remove(path string) error { return nil }
// WatchUntil watches the specified path for changes and blocks until ctx is canceled.
// eventHandler() must be non-nil, and pollInterval must be greater than 0.
// eventHandler() is invoked whenever a change event is observed or pollInterval elapses.
// errorHandler() is invoked (if non-nil) whenever an error occurs initializing or watching the specified path.
//
// If path is a directory, only the directory and immediate children are watched.
//
// If path does not exist or cannot be watched, an error is passed to errorHandler() and eventHandler() is called at pollInterval.
//
// Multiple observed events may collapse to a single invocation of eventHandler().
//
// eventHandler() is invoked immediately after successful initialization of the filesystem watch,
// in case the path changed concurrent with calling WatchUntil().
func WatchUntil(ctx context.Context, pollInterval time.Duration, path string, eventHandler func(), errorHandler func(err error)) {
if pollInterval <= 0 {
panic(fmt.Errorf("pollInterval must be > 0"))
}
if eventHandler == nil {
panic(fmt.Errorf("eventHandler must be non-nil"))
}
if errorHandler == nil {
errorHandler = func(err error) {}
}
// Initialize watcher, fall back to no-op
var (
eventsCh chan fsnotify.Event
errorCh chan error
watcher watchAddRemover
)
if w, err := fsnotify.NewWatcher(); err != nil {
errorHandler(fmt.Errorf("error creating file watcher, falling back to poll at interval %s: %w", pollInterval, err))
watcher = noopWatcher{}
} else {
watcher = w
eventsCh = w.Events
errorCh = w.Errors
defer func() {
_ = w.Close()
}()
}
// Initialize background poll
t := time.NewTicker(pollInterval)
defer t.Stop()
attemptPeriodicRewatch := false
// Start watching the path
if err := watcher.Add(path); err != nil {
errorHandler(err)
attemptPeriodicRewatch = true
} else {
// Invoke handle() at least once after successfully registering the listener,
// in case the file changed concurrent with calling WatchUntil.
eventHandler()
}
for {
select {
case <-ctx.Done():
return
case <-t.C:
// Prioritize exiting if context is canceled
if ctx.Err() != nil {
return
}
// Try to re-establish the watcher if we previously got a watch error
if attemptPeriodicRewatch {
_ = watcher.Remove(path)
if err := watcher.Add(path); err != nil {
errorHandler(err)
} else {
attemptPeriodicRewatch = false
}
}
// Handle
eventHandler()
case e := <-eventsCh:
// Prioritize exiting if context is canceled
if ctx.Err() != nil {
return
}
// Try to re-establish the watcher for events which dropped the existing watch
if e.Name == path && (e.Has(fsnotify.Remove) || e.Has(fsnotify.Rename)) {
_ = watcher.Remove(path)
if err := watcher.Add(path); err != nil {
errorHandler(err)
attemptPeriodicRewatch = true
}
}
// Handle
eventHandler()
case err := <-errorCh:
// Prioritize exiting if context is canceled
if ctx.Err() != nil {
return
}
// If the error occurs in response to calling watcher.Add, re-adding here could hot-loop.
// The periodic poll will attempt to re-establish the watch.
errorHandler(err)
attemptPeriodicRewatch = true
}
}
}