From 65a2794e60f199cc15e2d0af31d4e0c05d7d459a Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 26 May 2020 17:24:10 +0200 Subject: [PATCH] ci: add 'make test' target to validate CI job scripts This merges several files from the master branch into ci/centos so that the scripts, MarkDown and yaml files can be checked. Run 'make' or 'make test' to validate the contents of this branch. Signed-off-by: Niels de Vos --- .commitlintrc.yml | 30 +++++++++++++ .gitignore | 1 + Makefile | 71 ++++++++++++++++++++++++++++++ scripts/Dockerfile.test | 47 ++++++++++++++++++++ scripts/lint-extras.sh | 96 +++++++++++++++++++++++++++++++++++++++++ scripts/mdl-style.rb | 9 ++++ 6 files changed, 254 insertions(+) create mode 100644 .commitlintrc.yml create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 scripts/Dockerfile.test create mode 100755 scripts/lint-extras.sh create mode 100644 scripts/mdl-style.rb diff --git a/.commitlintrc.yml b/.commitlintrc.yml new file mode 100644 index 000000000..49b502ae1 --- /dev/null +++ b/.commitlintrc.yml @@ -0,0 +1,30 @@ +--- +# rules are documented at https://commitlint.js.org/#/reference-rules +# each rule has a n arry of three values: +# - 0, 1, 2 (rule is disabled, warning or error) +# - always/never +# - value +rules: + # the subject may not exceed 72 characters + header-max-length: [2, always, 72] + # the subject may not end with a "." + header-full-stop: [2, never, "."] + # we do not use scopes for commit messages + scope-empty: [1, always] + # valid types/prefixes, see docs/development-guide.md + type-enum: + - 1 + - always + - - build + - cephfs + - ci + - cleanup + - deploy + - doc + - e2e + - helm + - journal + - rbd + - rebase + - revert + - util diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..e62b9894a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.test-container-id diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..51a6280b8 --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +# Copyright 2020 The Ceph-CSI Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CONTAINER_CMD?=$(shell docker version >/dev/null 2>&1 && echo docker) +ifeq ($(CONTAINER_CMD),) + CONTAINER_CMD=$(shell podman version >/dev/null 2>&1 && echo podman) +endif +CPUS?=$(shell nproc --ignore=1) +CPUSET?=--cpuset-cpus=0-${CPUS} + +CSI_IMAGE_NAME=$(if $(ENV_CSI_IMAGE_NAME),$(ENV_CSI_IMAGE_NAME),quay.io/cephcsi/cephcsi) + +# passing TARGET=static-check on the 'make containerized-test' commandline will +# run the selected target instead of 'make test' in the container. Obviously +# other targets can be passed as well, making it easier for developers to run +# single tests. +TARGET ?= lint-all + +# Pass GIT_SINCE for the range of commits to test. Used with the commitlint +# target. +GIT_SINCE := origin/ci/centos + +SELINUX := $(shell getenforce 2>/dev/null) +ifeq ($(SELINUX),Enforcing) + SELINUX_VOL_FLAG = :z +endif + +.PHONY: test +test: + $(MAKE) containerized-test TARGET=lint-all + $(MAKE) containerized-test TARGET=commitlint + +.PHONY: lint-all lint-shell lint-markdown lint-yaml commitlint +lint-all: lint-shell lint-markdown lint-yaml + +lint-shell: + ./scripts/lint-extras.sh lint-shell + +lint-markdown: + ./scripts/lint-extras.sh lint-markdown + +lint-yaml: + ./scripts/lint-extras.sh lint-yaml + +commitlint: + commitlint --from $(GIT_SINCE) + +.PHONY: containerized-test +containerized-test: .test-container-id + $(CONTAINER_CMD) run --rm -v $(PWD):/go/src/github.com/ceph/ceph-csi$(SELINUX_VOL_FLAG) $(CSI_IMAGE_NAME):test make $(TARGET) GIT_SINCE=$(GIT_SINCE) + +# create a (cached) container image with dependencies for testing the CI jobs +.test-container-id: scripts/Dockerfile.test + [ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test + $(CONTAINER_CMD) build $(CPUSET) -t $(CSI_IMAGE_NAME):test -f ./scripts/Dockerfile.test . + $(CONTAINER_CMD) inspect -f '{{.Id}}' $(CSI_IMAGE_NAME):test > .test-container-id + +clean: + [ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test + $(RM) .test-container-id diff --git a/scripts/Dockerfile.test b/scripts/Dockerfile.test new file mode 100644 index 000000000..9f7e133a9 --- /dev/null +++ b/scripts/Dockerfile.test @@ -0,0 +1,47 @@ +# Container image for running the Ceph-CSI tests +# +# This container is based on Fedora so that recent versions of tools can easily +# be installed. +# +# Production containers are based one ceph/ceph:latest, which use CentOS as +# Operating System, so generated binaries and versions of dependencies may be a +# little different. +# + +FROM fedora:latest + +ARG GOLANGCI_VERSION=v1.21.0 +ARG GOSEC_VERSION=2.0.0 +ARG GOPATH=/go + +ENV \ + GOPATH=${GOPATH} \ + GO111MODULE=on \ + PATH="${GOPATH}/bin:/opt/commitlint/node_modules/.bin:${PATH}" + +RUN dnf -y install \ + git \ + make \ + golang \ + gcc \ + librados-devel \ + librbd-devel \ + rubygems \ + ShellCheck \ + yamllint \ + npm \ + && dnf -y update \ + && dnf -y clean all \ + && gem install mdl \ + && curl -sf "https://install.goreleaser.com/github.com/golangci/golangci-lint.sh" \ + | bash -s -- -b ${GOPATH}/bin "${GOLANGCI_VERSION}" \ + && curl -sfL "https://raw.githubusercontent.com/securego/gosec/master/install.sh" \ + | sh -s -- -b $GOPATH/bin "${GOSEC_VERSION}" \ + && curl -L https://git.io/get_helm.sh | bash \ + && mkdir /opt/commitlint && pushd /opt/commitlint \ + && npm init -y \ + && npm install @commitlint/cli \ + && popd \ + && true + +WORKDIR /go/src/github.com/ceph/ceph-csi diff --git a/scripts/lint-extras.sh b/scripts/lint-extras.sh new file mode 100755 index 000000000..569f23a86 --- /dev/null +++ b/scripts/lint-extras.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# vim: set ts=4 sw=4 et : + +# This script will be used to lint non-go files +# Usage: ./scripts/lint-extras.sh +# Available commands are [lint-shell lint-yaml lint-markdown lint-helmlint-all ] +set -e + +# Run checks from root of the repo +scriptdir="$(dirname "$(realpath "$0")")" +cd "$scriptdir/.." + +# run_check [optional args to checker...] +# Pass empty regex when no regex is needed +function run_check() { + local regex="$1" + shift + local exe="$1" + shift + + if [ -x "$(command -v "${exe}")" ]; then + if [ -z "${regex}" ]; then + "$exe" "$@" + else + find . -path ./vendor -prune -o -regextype egrep -iregex "$regex" -print0 | + xargs -0rt -n1 "$exe" "$@" + fi + elif [ "$all_required" -eq 0 ]; then + echo "Warning: $exe not found... skipping some tests." + else + echo "FAILED: All checks required, but $exe not found!" + exit 1 + fi +} + +all_required=0 + +function lint_markdown() { + # markdownlint: https://github.com/markdownlint/markdownlint + # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md + # Install via: gem install mdl + run_check '.*\.md' mdl --style scripts/mdl-style.rb +} + +function lint_shell() { + # Install via: dnf install shellcheck + run_check '.*\.(ba)?sh' shellcheck --external-sources + run_check '.*\.(ba)?sh' bash -n +} + +function lint_yaml() { + # Install via: pip install yamllint + # disable yamlint check for helm charts + run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: charts/*/templates/*.yaml}" +} + +function lint_helm() { + # Install via: https://github.com/helm/helm/blob/master/docs/install.md + run_check '' helm lint --namespace=test charts/* +} + +function lint_all() { + # runs all checks + all_required=1 + lint_shell + lint_yaml + lint_markdown + lint_helm +} +case "${1:-}" in +lint-shell) + lint_shell + ;; +lint-yaml) + lint_yaml + ;; +lint-markdown) + lint_markdown + ;; +lint-helm) + lint_helm + ;; +lint-all) + lint_all + ;; +*) + echo " $0 [command] +Available Commands: + lint-shell Lint shell files + lint-yaml Lint yaml files + lint-markdown Lint markdown files + lint-helm Lint helm charts + lint-all Run lint on all non-go files +" >&2 + ;; +esac diff --git a/scripts/mdl-style.rb b/scripts/mdl-style.rb new file mode 100644 index 000000000..3f392568c --- /dev/null +++ b/scripts/mdl-style.rb @@ -0,0 +1,9 @@ +all + +#Refer below url for more information about the markdown rules. +#https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md + +rule 'MD013', :code_blocks => false, :tables => false + +exclude_rule 'MD040' # Fenced code blocks should have a language specified +exclude_rule 'MD041' # First line in file should be a top level header