ci: add 'make test' target to validate CI job scripts

This merges several files from the master branch into ci/centos so that
the scripts, MarkDown and yaml files can be checked.

Run 'make' or 'make test' to validate the contents of this branch.

Signed-off-by: Niels de Vos <ndevos@redhat.com>

.commitlintrc.yml

@@ -0,0 +1,30 @@
+---
+# rules are documented at https://commitlint.js.org/#/reference-rules
+# each rule has a n arry of three values:
+# - 0, 1, 2 (rule is disabled, warning or error)
+# - always/never
+# - value
+rules:
+  # the subject may not exceed 72 characters
+  header-max-length: [2, always, 72]
+  # the subject may not end with a "."
+  header-full-stop: [2, never, "."]
+  # we do not use scopes for commit messages
+  scope-empty: [1, always]
+  # valid types/prefixes, see docs/development-guide.md
+  type-enum:
+    - 1
+    - always
+    - - build
+      - cephfs
+      - ci
+      - cleanup
+      - deploy
+      - doc
+      - e2e
+      - helm
+      - journal
+      - rbd
+      - rebase
+      - revert
+      - util

.gitignore

@@ -0,0 +1 @@
+.test-container-id

Makefile

@@ -0,0 +1,71 @@
+# Copyright 2020 The Ceph-CSI Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CONTAINER_CMD?=$(shell docker version >/dev/null 2>&1 && echo docker)
+ifeq ($(CONTAINER_CMD),)
+    CONTAINER_CMD=$(shell podman version >/dev/null 2>&1 && echo podman)
+endif
+CPUS?=$(shell nproc --ignore=1)
+CPUSET?=--cpuset-cpus=0-${CPUS}
+
+CSI_IMAGE_NAME=$(if $(ENV_CSI_IMAGE_NAME),$(ENV_CSI_IMAGE_NAME),quay.io/cephcsi/cephcsi)
+
+# passing TARGET=static-check on the 'make containerized-test' commandline will
+# run the selected target instead of 'make test' in the container. Obviously
+# other targets can be passed as well, making it easier for developers to run
+# single tests.
+TARGET ?= lint-all
+
+# Pass GIT_SINCE for the range of commits to test. Used with the commitlint
+# target.
+GIT_SINCE := origin/ci/centos
+
+SELINUX := $(shell getenforce 2>/dev/null)
+ifeq ($(SELINUX),Enforcing)
+	SELINUX_VOL_FLAG = :z
+endif
+
+.PHONY: test
+test:
+	$(MAKE) containerized-test TARGET=lint-all
+	$(MAKE) containerized-test TARGET=commitlint
+
+.PHONY: lint-all lint-shell lint-markdown lint-yaml commitlint
+lint-all: lint-shell lint-markdown lint-yaml
+
+lint-shell:
+	./scripts/lint-extras.sh lint-shell
+
+lint-markdown:
+	./scripts/lint-extras.sh lint-markdown
+
+lint-yaml:
+	./scripts/lint-extras.sh lint-yaml
+
+commitlint:
+	commitlint --from $(GIT_SINCE)
+
+.PHONY: containerized-test
+containerized-test: .test-container-id
+	$(CONTAINER_CMD) run --rm -v $(PWD):/go/src/github.com/ceph/ceph-csi$(SELINUX_VOL_FLAG) $(CSI_IMAGE_NAME):test make $(TARGET) GIT_SINCE=$(GIT_SINCE)
+
+# create a (cached) container image with dependencies for testing the CI jobs
+.test-container-id: scripts/Dockerfile.test
+	[ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test
+	$(CONTAINER_CMD) build $(CPUSET) -t $(CSI_IMAGE_NAME):test -f ./scripts/Dockerfile.test .
+	$(CONTAINER_CMD) inspect -f '{{.Id}}' $(CSI_IMAGE_NAME):test > .test-container-id
+
+clean:
+	[ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test
+	$(RM) .test-container-id

scripts/Dockerfile.test

@@ -0,0 +1,47 @@
+# Container image for running the Ceph-CSI tests
+#
+# This container is based on Fedora so that recent versions of tools can easily
+# be installed.
+#
+# Production containers are based one ceph/ceph:latest, which use CentOS as
+# Operating System, so generated binaries and versions of dependencies may be a
+# little different.
+#
+
+FROM fedora:latest
+
+ARG GOLANGCI_VERSION=v1.21.0
+ARG GOSEC_VERSION=2.0.0
+ARG GOPATH=/go
+
+ENV \
+ GOPATH=${GOPATH} \
+ GO111MODULE=on \
+ PATH="${GOPATH}/bin:/opt/commitlint/node_modules/.bin:${PATH}"
+
+RUN dnf -y install \
+	git \
+	make \
+	golang \
+	gcc \
+	librados-devel \
+	librbd-devel \
+	rubygems \
+	ShellCheck \
+	yamllint \
+	npm \
+    && dnf -y update \
+    && dnf -y clean all \
+    && gem install mdl \
+    && curl -sf "https://install.goreleaser.com/github.com/golangci/golangci-lint.sh" \
+       | bash -s -- -b ${GOPATH}/bin "${GOLANGCI_VERSION}" \
+    && curl -sfL "https://raw.githubusercontent.com/securego/gosec/master/install.sh" \
+       | sh -s -- -b $GOPATH/bin "${GOSEC_VERSION}" \
+    && curl -L https://git.io/get_helm.sh | bash \
+    && mkdir /opt/commitlint && pushd /opt/commitlint \
+    && npm init -y \
+    && npm install @commitlint/cli \
+    && popd \
+    && true
+
+WORKDIR /go/src/github.com/ceph/ceph-csi

scripts/lint-extras.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# vim: set ts=4 sw=4 et :
+
+# This script will be used to lint non-go files
+# Usage: ./scripts/lint-extras.sh <command>
+# Available commands are [lint-shell lint-yaml lint-markdown lint-helmlint-all ]
+set -e
+
+# Run checks from root of the repo
+scriptdir="$(dirname "$(realpath "$0")")"
+cd "$scriptdir/.."
+
+# run_check <file_regex> <checker_exe> [optional args to checker...]
+# Pass empty regex when no regex is needed
+function run_check() {
+    local regex="$1"
+    shift
+    local exe="$1"
+    shift
+
+    if [ -x "$(command -v "${exe}")" ]; then
+        if [ -z "${regex}" ]; then
+            "$exe" "$@"
+        else
+            find . -path ./vendor -prune -o -regextype egrep -iregex "$regex" -print0 |
+                xargs -0rt -n1 "$exe" "$@"
+        fi
+    elif [ "$all_required" -eq 0 ]; then
+        echo "Warning: $exe not found... skipping some tests."
+    else
+        echo "FAILED: All checks required, but $exe not found!"
+        exit 1
+    fi
+}
+
+all_required=0
+
+function lint_markdown() {
+    # markdownlint: https://github.com/markdownlint/markdownlint
+    # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
+    # Install via: gem install mdl
+    run_check '.*\.md' mdl --style scripts/mdl-style.rb
+}
+
+function lint_shell() {
+    # Install via: dnf install shellcheck
+    run_check '.*\.(ba)?sh' shellcheck --external-sources
+    run_check '.*\.(ba)?sh' bash -n
+}
+
+function lint_yaml() {
+    # Install via: pip install yamllint
+    # disable yamlint check for helm charts
+    run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: charts/*/templates/*.yaml}"
+}
+
+function lint_helm() {
+    # Install via: https://github.com/helm/helm/blob/master/docs/install.md
+    run_check '' helm lint --namespace=test charts/*
+}
+
+function lint_all() {
+    # runs all checks
+    all_required=1
+    lint_shell
+    lint_yaml
+    lint_markdown
+    lint_helm
+}
+case "${1:-}" in
+lint-shell)
+    lint_shell
+    ;;
+lint-yaml)
+    lint_yaml
+    ;;
+lint-markdown)
+    lint_markdown
+    ;;
+lint-helm)
+    lint_helm
+    ;;
+lint-all)
+    lint_all
+    ;;
+*)
+    echo " $0 [command]
+Available Commands:
+  lint-shell             Lint shell files
+  lint-yaml              Lint yaml files
+  lint-markdown          Lint markdown files
+  lint-helm              Lint helm charts
+  lint-all               Run lint on all non-go files
+" >&2
+    ;;
+esac

scripts/mdl-style.rb

@@ -0,0 +1,9 @@
+all
+
+#Refer below url for more information about the markdown rules.
+#https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
+
+rule 'MD013', :code_blocks => false, :tables => false
+
+exclude_rule 'MD040' # Fenced code blocks should have a language specified
+exclude_rule 'MD041' # First line in file should be a top level header