build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>

e2e/vendor/github.com/google/gnostic-models/compiler/README.md

@@ -0,0 +1,4 @@
+# Compiler support code
+
+This directory contains compiler support code used by Gnostic and Gnostic
+extensions.

e2e/vendor/github.com/google/gnostic-models/compiler/context.go

@@ -0,0 +1,49 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Context contains state of the compiler as it traverses a document.
+type Context struct {
+	Parent            *Context
+	Name              string
+	Node              *yaml.Node
+	ExtensionHandlers *[]ExtensionHandler
+}
+
+// NewContextWithExtensions returns a new object representing the compiler state
+func NewContextWithExtensions(name string, node *yaml.Node, parent *Context, extensionHandlers *[]ExtensionHandler) *Context {
+	return &Context{Name: name, Node: node, Parent: parent, ExtensionHandlers: extensionHandlers}
+}
+
+// NewContext returns a new object representing the compiler state
+func NewContext(name string, node *yaml.Node, parent *Context) *Context {
+	if parent != nil {
+		return &Context{Name: name, Node: node, Parent: parent, ExtensionHandlers: parent.ExtensionHandlers}
+	}
+	return &Context{Name: name, Parent: parent, ExtensionHandlers: nil}
+}
+
+// Description returns a text description of the compiler state
+func (context *Context) Description() string {
+	name := context.Name
+	if context.Parent != nil {
+		name = context.Parent.Description() + "." + name
+	}
+	return name
+}

e2e/vendor/github.com/google/gnostic-models/compiler/error.go

@@ -0,0 +1,70 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import "fmt"
+
+// Error represents compiler errors and their location in the document.
+type Error struct {
+	Context *Context
+	Message string
+}
+
+// NewError creates an Error.
+func NewError(context *Context, message string) *Error {
+	return &Error{Context: context, Message: message}
+}
+
+func (err *Error) locationDescription() string {
+	if err.Context.Node != nil {
+		return fmt.Sprintf("[%d,%d] %s", err.Context.Node.Line, err.Context.Node.Column, err.Context.Description())
+	}
+	return err.Context.Description()
+}
+
+// Error returns the string value of an Error.
+func (err *Error) Error() string {
+	if err.Context == nil {
+		return err.Message
+	}
+	return err.locationDescription() + " " + err.Message
+}
+
+// ErrorGroup is a container for groups of Error values.
+type ErrorGroup struct {
+	Errors []error
+}
+
+// NewErrorGroupOrNil returns a new ErrorGroup for a slice of errors or nil if the slice is empty.
+func NewErrorGroupOrNil(errors []error) error {
+	if len(errors) == 0 {
+		return nil
+	} else if len(errors) == 1 {
+		return errors[0]
+	} else {
+		return &ErrorGroup{Errors: errors}
+	}
+}
+
+func (group *ErrorGroup) Error() string {
+	result := ""
+	for i, err := range group.Errors {
+		if i > 0 {
+			result += "\n"
+		}
+		result += err.Error()
+	}
+	return result
+}

e2e/vendor/github.com/google/gnostic-models/compiler/extensions.go

@@ -0,0 +1,86 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+	yaml "gopkg.in/yaml.v3"
+
+	extensions "github.com/google/gnostic-models/extensions"
+)
+
+// ExtensionHandler describes a binary that is called by the compiler to handle specification extensions.
+type ExtensionHandler struct {
+	Name string
+}
+
+// CallExtension calls a binary extension handler.
+func CallExtension(context *Context, in *yaml.Node, extensionName string) (handled bool, response *anypb.Any, err error) {
+	if context == nil || context.ExtensionHandlers == nil {
+		return false, nil, nil
+	}
+	handled = false
+	for _, handler := range *(context.ExtensionHandlers) {
+		response, err = handler.handle(in, extensionName)
+		if response == nil {
+			continue
+		} else {
+			handled = true
+			break
+		}
+	}
+	return handled, response, err
+}
+
+func (extensionHandlers *ExtensionHandler) handle(in *yaml.Node, extensionName string) (*anypb.Any, error) {
+	if extensionHandlers.Name != "" {
+		yamlData, _ := yaml.Marshal(in)
+		request := &extensions.ExtensionHandlerRequest{
+			CompilerVersion: &extensions.Version{
+				Major: 0,
+				Minor: 1,
+				Patch: 0,
+			},
+			Wrapper: &extensions.Wrapper{
+				Version:       "unknown", // TODO: set this to the type/version of spec being parsed.
+				Yaml:          string(yamlData),
+				ExtensionName: extensionName,
+			},
+		}
+		requestBytes, _ := proto.Marshal(request)
+		cmd := exec.Command(extensionHandlers.Name)
+		cmd.Stdin = bytes.NewReader(requestBytes)
+		output, err := cmd.Output()
+		if err != nil {
+			return nil, err
+		}
+		response := &extensions.ExtensionHandlerResponse{}
+		err = proto.Unmarshal(output, response)
+		if err != nil || !response.Handled {
+			return nil, err
+		}
+		if len(response.Errors) != 0 {
+			return nil, fmt.Errorf("Errors when parsing: %+v for field %s by vendor extension handler %s. Details %+v", in, extensionName, extensionHandlers.Name, strings.Join(response.Errors, ","))
+		}
+		return response.Value, nil
+	}
+	return nil, nil
+}

e2e/vendor/github.com/google/gnostic-models/compiler/helpers.go

@@ -0,0 +1,397 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/google/gnostic-models/jsonschema"
+)
+
+// compiler helper functions, usually called from generated code
+
+// UnpackMap gets a *yaml.Node if possible.
+func UnpackMap(in *yaml.Node) (*yaml.Node, bool) {
+	if in == nil {
+		return nil, false
+	}
+	return in, true
+}
+
+// SortedKeysForMap returns the sorted keys of a yamlv2.MapSlice.
+func SortedKeysForMap(m *yaml.Node) []string {
+	keys := make([]string, 0)
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			keys = append(keys, m.Content[i].Value)
+		}
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// MapHasKey returns true if a yamlv2.MapSlice contains a specified key.
+func MapHasKey(m *yaml.Node, key string) bool {
+	if m == nil {
+		return false
+	}
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			itemKey := m.Content[i].Value
+			if key == itemKey {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// MapValueForKey gets the value of a map value for a specified key.
+func MapValueForKey(m *yaml.Node, key string) *yaml.Node {
+	if m == nil {
+		return nil
+	}
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			itemKey := m.Content[i].Value
+			if key == itemKey {
+				return m.Content[i+1]
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertInterfaceArrayToStringArray converts an array of interfaces to an array of strings, if possible.
+func ConvertInterfaceArrayToStringArray(interfaceArray []interface{}) []string {
+	stringArray := make([]string, 0)
+	for _, item := range interfaceArray {
+		v, ok := item.(string)
+		if ok {
+			stringArray = append(stringArray, v)
+		}
+	}
+	return stringArray
+}
+
+// SequenceNodeForNode returns a node if it is a SequenceNode.
+func SequenceNodeForNode(node *yaml.Node) (*yaml.Node, bool) {
+	if node.Kind != yaml.SequenceNode {
+		return nil, false
+	}
+	return node, true
+}
+
+// BoolForScalarNode returns the bool value of a node.
+func BoolForScalarNode(node *yaml.Node) (bool, bool) {
+	if node == nil {
+		return false, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return BoolForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return false, false
+	}
+	if node.Tag != "!!bool" {
+		return false, false
+	}
+	v, err := strconv.ParseBool(node.Value)
+	if err != nil {
+		return false, false
+	}
+	return v, true
+}
+
+// IntForScalarNode returns the integer value of a node.
+func IntForScalarNode(node *yaml.Node) (int64, bool) {
+	if node == nil {
+		return 0, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return IntForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return 0, false
+	}
+	if node.Tag != "!!int" {
+		return 0, false
+	}
+	v, err := strconv.ParseInt(node.Value, 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	return v, true
+}
+
+// FloatForScalarNode returns the float value of a node.
+func FloatForScalarNode(node *yaml.Node) (float64, bool) {
+	if node == nil {
+		return 0.0, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return FloatForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return 0.0, false
+	}
+	if (node.Tag != "!!int") && (node.Tag != "!!float") {
+		return 0.0, false
+	}
+	v, err := strconv.ParseFloat(node.Value, 64)
+	if err != nil {
+		return 0.0, false
+	}
+	return v, true
+}
+
+// StringForScalarNode returns the string value of a node.
+func StringForScalarNode(node *yaml.Node) (string, bool) {
+	if node == nil {
+		return "", false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return StringForScalarNode(node.Content[0])
+	}
+	switch node.Kind {
+	case yaml.ScalarNode:
+		switch node.Tag {
+		case "!!int":
+			return node.Value, true
+		case "!!str":
+			return node.Value, true
+		case "!!timestamp":
+			return node.Value, true
+		case "!!null":
+			return "", true
+		default:
+			return "", false
+		}
+	default:
+		return "", false
+	}
+}
+
+// StringArrayForSequenceNode converts a sequence node to an array of strings, if possible.
+func StringArrayForSequenceNode(node *yaml.Node) []string {
+	stringArray := make([]string, 0)
+	for _, item := range node.Content {
+		v, ok := StringForScalarNode(item)
+		if ok {
+			stringArray = append(stringArray, v)
+		}
+	}
+	return stringArray
+}
+
+// MissingKeysInMap identifies which keys from a list of required keys are not in a map.
+func MissingKeysInMap(m *yaml.Node, requiredKeys []string) []string {
+	missingKeys := make([]string, 0)
+	for _, k := range requiredKeys {
+		if !MapHasKey(m, k) {
+			missingKeys = append(missingKeys, k)
+		}
+	}
+	return missingKeys
+}
+
+// InvalidKeysInMap returns keys in a map that don't match a list of allowed keys and patterns.
+func InvalidKeysInMap(m *yaml.Node, allowedKeys []string, allowedPatterns []*regexp.Regexp) []string {
+	invalidKeys := make([]string, 0)
+	if m == nil || m.Kind != yaml.MappingNode {
+		return invalidKeys
+	}
+	for i := 0; i < len(m.Content); i += 2 {
+		key := m.Content[i].Value
+		found := false
+		// does the key match an allowed key?
+		for _, allowedKey := range allowedKeys {
+			if key == allowedKey {
+				found = true
+				break
+			}
+		}
+		if !found {
+			// does the key match an allowed pattern?
+			for _, allowedPattern := range allowedPatterns {
+				if allowedPattern.MatchString(key) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				invalidKeys = append(invalidKeys, key)
+			}
+		}
+	}
+	return invalidKeys
+}
+
+// NewNullNode creates a new Null node.
+func NewNullNode() *yaml.Node {
+	node := &yaml.Node{
+		Kind: yaml.ScalarNode,
+		Tag:  "!!null",
+	}
+	return node
+}
+
+// NewMappingNode creates a new Mapping node.
+func NewMappingNode() *yaml.Node {
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Content: make([]*yaml.Node, 0),
+	}
+}
+
+// NewSequenceNode creates a new Sequence node.
+func NewSequenceNode() *yaml.Node {
+	node := &yaml.Node{
+		Kind:    yaml.SequenceNode,
+		Content: make([]*yaml.Node, 0),
+	}
+	return node
+}
+
+// NewScalarNodeForString creates a new node to hold a string.
+func NewScalarNodeForString(s string) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!str",
+		Value: s,
+	}
+}
+
+// NewSequenceNodeForStringArray creates a new node to hold an array of strings.
+func NewSequenceNodeForStringArray(strings []string) *yaml.Node {
+	node := &yaml.Node{
+		Kind:    yaml.SequenceNode,
+		Content: make([]*yaml.Node, 0),
+	}
+	for _, s := range strings {
+		node.Content = append(node.Content, NewScalarNodeForString(s))
+	}
+	return node
+}
+
+// NewScalarNodeForBool creates a new node to hold a bool.
+func NewScalarNodeForBool(b bool) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!bool",
+		Value: fmt.Sprintf("%t", b),
+	}
+}
+
+// NewScalarNodeForFloat creates a new node to hold a float.
+func NewScalarNodeForFloat(f float64) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!float",
+		Value: fmt.Sprintf("%g", f),
+	}
+}
+
+// NewScalarNodeForInt creates a new node to hold an integer.
+func NewScalarNodeForInt(i int64) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!int",
+		Value: fmt.Sprintf("%d", i),
+	}
+}
+
+// PluralProperties returns the string "properties" pluralized.
+func PluralProperties(count int) string {
+	if count == 1 {
+		return "property"
+	}
+	return "properties"
+}
+
+// StringArrayContainsValue returns true if a string array contains a specified value.
+func StringArrayContainsValue(array []string, value string) bool {
+	for _, item := range array {
+		if item == value {
+			return true
+		}
+	}
+	return false
+}
+
+// StringArrayContainsValues returns true if a string array contains all of a list of specified values.
+func StringArrayContainsValues(array []string, values []string) bool {
+	for _, value := range values {
+		if !StringArrayContainsValue(array, value) {
+			return false
+		}
+	}
+	return true
+}
+
+// StringValue returns the string value of an item.
+func StringValue(item interface{}) (value string, ok bool) {
+	value, ok = item.(string)
+	if ok {
+		return value, ok
+	}
+	intValue, ok := item.(int)
+	if ok {
+		return strconv.Itoa(intValue), true
+	}
+	return "", false
+}
+
+// Description returns a human-readable represention of an item.
+func Description(item interface{}) string {
+	value, ok := item.(*yaml.Node)
+	if ok {
+		return jsonschema.Render(value)
+	}
+	return fmt.Sprintf("%+v", item)
+}
+
+// Display returns a description of a node for use in error messages.
+func Display(node *yaml.Node) string {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		switch node.Tag {
+		case "!!str":
+			return fmt.Sprintf("%s (string)", node.Value)
+		}
+	}
+	return fmt.Sprintf("%+v (%T)", node, node)
+}
+
+// Marshal creates a yaml version of a structure in our preferred style
+func Marshal(in *yaml.Node) []byte {
+	clearStyle(in)
+	//bytes, _ := yaml.Marshal(&yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{in}})
+	bytes, _ := yaml.Marshal(in)
+
+	return bytes
+}
+
+func clearStyle(node *yaml.Node) {
+	node.Style = 0
+	for _, c := range node.Content {
+		clearStyle(c)
+	}
+}

e2e/vendor/github.com/google/gnostic-models/compiler/main.go

@@ -0,0 +1,16 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package compiler provides support functions to generated compiler code.
+package compiler

e2e/vendor/github.com/google/gnostic-models/compiler/reader.go

@@ -0,0 +1,307 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+var verboseReader = false
+
+var fileCache map[string][]byte
+var infoCache map[string]*yaml.Node
+
+var fileCacheEnable = true
+var infoCacheEnable = true
+
+// These locks are used to synchronize accesses to the fileCache and infoCache
+// maps (above). They are global state and can throw thread-related errors
+// when modified from separate goroutines. The general strategy is to protect
+// all public functions in this file with mutex Lock() calls. As a result, to
+// avoid deadlock, these public functions should not call other public
+// functions, so some public functions have private equivalents.
+// In the future, we might consider replacing the maps with sync.Map and
+// eliminating these mutexes.
+var fileCacheMutex sync.Mutex
+var infoCacheMutex sync.Mutex
+
+func initializeFileCache() {
+	if fileCache == nil {
+		fileCache = make(map[string][]byte, 0)
+	}
+}
+
+func initializeInfoCache() {
+	if infoCache == nil {
+		infoCache = make(map[string]*yaml.Node, 0)
+	}
+}
+
+// EnableFileCache turns on file caching.
+func EnableFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCacheEnable = true
+}
+
+// EnableInfoCache turns on parsed info caching.
+func EnableInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCacheEnable = true
+}
+
+// DisableFileCache turns off file caching.
+func DisableFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCacheEnable = false
+}
+
+// DisableInfoCache turns off parsed info caching.
+func DisableInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCacheEnable = false
+}
+
+// RemoveFromFileCache removes an entry from the file cache.
+func RemoveFromFileCache(fileurl string) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	if !fileCacheEnable {
+		return
+	}
+	initializeFileCache()
+	delete(fileCache, fileurl)
+}
+
+// RemoveFromInfoCache removes an entry from the info cache.
+func RemoveFromInfoCache(filename string) {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	if !infoCacheEnable {
+		return
+	}
+	initializeInfoCache()
+	delete(infoCache, filename)
+}
+
+// GetInfoCache returns the info cache map.
+func GetInfoCache() map[string]*yaml.Node {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	if infoCache == nil {
+		initializeInfoCache()
+	}
+	return infoCache
+}
+
+// ClearFileCache clears the file cache.
+func ClearFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCache = make(map[string][]byte, 0)
+}
+
+// ClearInfoCache clears the info cache.
+func ClearInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCache = make(map[string]*yaml.Node)
+}
+
+// ClearCaches clears all caches.
+func ClearCaches() {
+	ClearFileCache()
+	ClearInfoCache()
+}
+
+// FetchFile gets a specified file from the local filesystem or a remote location.
+func FetchFile(fileurl string) ([]byte, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	return fetchFile(fileurl)
+}
+
+func fetchFile(fileurl string) ([]byte, error) {
+	var bytes []byte
+	initializeFileCache()
+	if fileCacheEnable {
+		bytes, ok := fileCache[fileurl]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit %s", fileurl)
+			}
+			return bytes, nil
+		}
+		if verboseReader {
+			log.Printf("Fetching %s", fileurl)
+		}
+	}
+	response, err := http.Get(fileurl)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("Error downloading %s: %s", fileurl, response.Status)
+	}
+	bytes, err = ioutil.ReadAll(response.Body)
+	if fileCacheEnable && err == nil {
+		fileCache[fileurl] = bytes
+	}
+	return bytes, err
+}
+
+// ReadBytesForFile reads the bytes of a file.
+func ReadBytesForFile(filename string) ([]byte, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	return readBytesForFile(filename)
+}
+
+func readBytesForFile(filename string) ([]byte, error) {
+	// is the filename a url?
+	fileurl, _ := url.Parse(filename)
+	if fileurl.Scheme != "" {
+		// yes, fetch it
+		bytes, err := fetchFile(filename)
+		if err != nil {
+			return nil, err
+		}
+		return bytes, nil
+	}
+	// no, it's a local filename
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, nil
+}
+
+// ReadInfoFromBytes unmarshals a file as a *yaml.Node.
+func ReadInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	return readInfoFromBytes(filename, bytes)
+}
+
+func readInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
+	initializeInfoCache()
+	if infoCacheEnable {
+		cachedInfo, ok := infoCache[filename]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit info for file %s", filename)
+			}
+			return cachedInfo, nil
+		}
+		if verboseReader {
+			log.Printf("Reading info for file %s", filename)
+		}
+	}
+	var info yaml.Node
+	err := yaml.Unmarshal(bytes, &info)
+	if err != nil {
+		return nil, err
+	}
+	if infoCacheEnable && len(filename) > 0 {
+		infoCache[filename] = &info
+	}
+	return &info, nil
+}
+
+// ReadInfoForRef reads a file and return the fragment needed to resolve a $ref.
+func ReadInfoForRef(basefile string, ref string) (*yaml.Node, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	initializeInfoCache()
+	if infoCacheEnable {
+		info, ok := infoCache[ref]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit for ref %s#%s", basefile, ref)
+			}
+			return info, nil
+		}
+		if verboseReader {
+			log.Printf("Reading info for ref %s#%s", basefile, ref)
+		}
+	}
+	basedir, _ := filepath.Split(basefile)
+	parts := strings.Split(ref, "#")
+	var filename string
+	if parts[0] != "" {
+		filename = parts[0]
+		if _, err := url.ParseRequestURI(parts[0]); err != nil {
+			// It is not an URL, so the file is local
+			filename = basedir + parts[0]
+		}
+	} else {
+		filename = basefile
+	}
+	bytes, err := readBytesForFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	info, err := readInfoFromBytes(filename, bytes)
+	if info != nil && info.Kind == yaml.DocumentNode {
+		info = info.Content[0]
+	}
+	if err != nil {
+		log.Printf("File error: %v\n", err)
+	} else {
+		if info == nil {
+			return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
+		}
+		if len(parts) > 1 {
+			path := strings.Split(parts[1], "/")
+			for i, key := range path {
+				if i > 0 {
+					m := info
+					if true {
+						found := false
+						for i := 0; i < len(m.Content); i += 2 {
+							if m.Content[i].Value == key {
+								info = m.Content[i+1]
+								found = true
+							}
+						}
+						if !found {
+							infoCache[ref] = nil
+							return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
+						}
+					}
+				}
+			}
+		}
+	}
+	if infoCacheEnable {
+		infoCache[ref] = info
+	}
+	return info, nil
+}