[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

.gitlab-ci.yml

@@ -77,6 +77,7 @@ ci-authorized:
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
+  - .gitlab-ci/shellcheck.yml
   - .gitlab-ci/terraform.yml
   - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml

.gitlab-ci/lint.yml

@@ -1,24 +1,13 @@
 ---
-generate-pre-commit:
+yamllint:
-  image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
+  extends: .job
-  stage: build
-  before_script: []
-  script:
-    - >
-      yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
-      sed 's/^/      - /' |
-      cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
-  artifacts:
-    paths:
-      - pre-commit-generated.yml
-
-run-pre-commit:
   stage: unit-tests
-  trigger:
+  tags: [light]
-    include:
+  variables:
-      - artifact: pre-commit-generated.yml
+    LANG: C.UTF-8
-        job: generate-pre-commit
+  script:
-    strategy: depend
+    - yamllint --strict .
+  except: ['triggers', 'master']
 
 vagrant-validate:
   extends: .job
@@ -30,11 +19,108 @@ vagrant-validate:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
 
+ansible-lint:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - ansible-lint -v
+  except: ['triggers', 'master']
+
+jinja-syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
+  except: ['triggers', 'master']
+
+syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_INVENTORY: inventory/local-tests.cfg
+    ANSIBLE_REMOTE_USER: root
+    ANSIBLE_BECOME: "true"
+    ANSIBLE_BECOME_USER: root
+    ANSIBLE_VERBOSITY: "3"
+  script:
+    - ansible-playbook --syntax-check cluster.yml
+    - ansible-playbook --syntax-check playbooks/cluster.yml
+    - ansible-playbook --syntax-check upgrade-cluster.yml
+    - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
+    - ansible-playbook --syntax-check reset.yml
+    - ansible-playbook --syntax-check playbooks/reset.yml
+    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
+  except: ['triggers', 'master']
+
+collection-build-install-sanity-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
+  script:
+    - ansible-galaxy collection build
+    - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+    - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  tags: [light]
+  extends: .job
+  before_script:
+    - ./tests/scripts/rebase.sh
+  script:
+    - pip3 install tox
+    - cd contrib/inventory_builder && tox
+  except: ['triggers', 'master']
+
+markdownlint:
+  stage: unit-tests
+  tags: [light]
+  image: node
+  before_script:
+    - npm install -g markdownlint-cli@0.22.0
+  script:
+    - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
+
+generate-sidebar:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - scripts/gen_docs_sidebar.sh
+    - git diff --exit-code
+
+check-readme-versions:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_readme_versions.sh
 
-# TODO: convert to pre-commit hook
 check-galaxy-version:
   stage: unit-tests
   tags: [light]
   image: python:3
   script:
     - tests/scripts/check_galaxy_version.sh
+
+check-typo:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_typo.sh
+
+ci-matrix:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/md-table/test.sh

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -1,17 +0,0 @@
----
-# stub pipeline for dynamic generation
-pre-commit:
-  tags:
-  - light
-  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
-  variables:
-    PRE_COMMIT_HOME: /pre-commit-cache
-  script:
-  - pre-commit run -a $HOOK_ID
-  cache:
-    key: pre-commit-$HOOK_ID
-    paths:
-    - /pre-commit-cache
-  parallel:
-    matrix:
-    - HOOK_ID:

.gitlab-ci/shellcheck.yml

@@ -0,0 +1,16 @@
+---
+shellcheck:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    SHELLCHECK_VERSION: v0.7.1
+  before_script:
+    - ./tests/scripts/rebase.sh
+    - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
+    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
+    - shellcheck --version
+  script:
+    # Run shellcheck for all *.sh
+    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
+  except: ['triggers', 'master']

.markdownlint.yaml

@@ -0,0 +1,3 @@
+---
+MD013: false
+MD029: false

.md_style.rb

@@ -1,4 +0,0 @@
-all
-exclude_rule 'MD013'
-exclude_rule 'MD029'
-rule 'MD007', :indent => 2

.mdlrc

@@ -1 +0,0 @@
-style "#{File.dirname(__FILE__)}/.md_style.rb"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v3.4.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,59 +15,47 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.35.1
+    rev: v1.27.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.12.0
+    rev: v0.11.0
     hooks:
       - id: markdownlint
-        exclude: "^.github|(^docs/_sidebar\\.md$)"
+        args: [-r, "~MD013,~MD029"]
+        exclude: "^.git"
 
-  - repo: https://github.com/shellcheck-py/shellcheck-py
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
-    rev: v0.10.0.1
+    rev: 3.0.0
     hooks:
       - id: shellcheck
-        args: ["--severity=error"]
+        args: [--severity, "error"]
         exclude: "^.git"
         files: "\\.sh$"
 
-  - repo: https://github.com/ansible/ansible-lint
-    rev: v24.5.0
-    hooks:
-      - id: ansible-lint
-        additional_dependencies:
-          - ansible==9.5.1
-          - jsonschema==4.22.0
-          - jmespath==1.0.1
-          - netaddr==1.2.1
-
-  - repo: https://github.com/VannTen/misspell
-    # Waiting on https://github.com/golangci/misspell/pull/19 to get merged
-    rev: 8592a4e
-    hooks:
-      - id: misspell
-        exclude: "OWNERS_ALIASES$"
-
   - repo: local
     hooks:
+      - id: ansible-lint
+        name: ansible-lint
+        entry: ansible-lint -v
+        language: python
+        pass_filenames: false
+        additional_dependencies:
+          - .[community]
+
       - id: ansible-syntax-check
         name: ansible-syntax-check
         entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
         language: python
         files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
-        additional_dependencies:
-          - ansible==9.5.1
 
       - id: tox-inventory-builder
         name: tox-inventory-builder
         entry: bash -c "cd contrib/inventory_builder && tox"
         language: python
         pass_filenames: false
-        additional_dependencies:
-          - tox==4.15.0
 
       - id: check-readme-versions
         name: check-readme-versions
@@ -75,14 +63,6 @@ repos:
         language: script
         pass_filenames: false
 
-      - id: collection-build-install
-        name: Build and install kubernetes-sigs.kubespray Ansible collection
-        language: python
-        additional_dependencies:
-          - ansible-core>=2.16.4
-        entry: tests/scripts/collection-build-install.sh
-        pass_filenames: false
-
       - id: generate-docs-sidebar
         name: generate-docs-sidebar
         entry: scripts/gen_docs_sidebar.sh
@@ -91,13 +71,9 @@ repos:
 
       - id: ci-matrix
         name: ci-matrix
-        entry: tests/scripts/md-table/main.py
+        entry: tests/scripts/md-table/test.sh
-        language: python
+        language: script
         pass_filenames: false
-        additional_dependencies:
-          - jinja2
-          - pathlib
-          - pyaml
 
       - id: jinja-syntax-check
         name: jinja-syntax-check
@@ -106,4 +82,4 @@ repos:
         types:
           - jinja
         additional_dependencies:
-          - jinja2
+          - Jinja2

README.md

@@ -1,3 +1,5 @@
+TEST CI
+
 # Deploy a Production Ready Kubernetes Cluster
 
 ![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)
@@ -142,7 +144,7 @@ vagrant up
 
 - **Flatcar Container Linux by Kinvolk**
 - **Debian** Bookworm, Bullseye, Buster
-- **Ubuntu** 20.04, 22.04, 24.04
+- **Ubuntu** 20.04, 22.04
 - **CentOS/RHEL** 7, [8, 9](docs/operating_systems/centos.md#centos-8)
 - **Fedora** 37, 38
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
@@ -162,7 +164,7 @@ Note: Upstart/SysV init based OS types are not supported.
 - Core
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.5
   - [etcd](https://github.com/etcd-io/etcd) v3.5.12
-  - [docker](https://www.docker.com/) v26.1
+  - [docker](https://www.docker.com/) v24.0 (see [Note](#container-runtime-notes))
   - [containerd](https://containerd.io/) v1.7.16
   - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
@@ -197,6 +199,7 @@ Note: Upstart/SysV init based OS types are not supported.
 
 ## Container Runtime Notes
 
+- Supported Docker versions are 18.09, 19.03, 20.10, 23.0 and 24.0. The *recommended* Docker version is 24.0. `Kubelet` might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. the YUM  ``versionlock`` plugin or ``apt pin``).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
 
 ## Requirements

contrib/terraform/nifcloud/README.md

@@ -72,7 +72,6 @@ The setup looks like following
 
   ```bash
   ./generate-inventory.sh > sample-inventory/inventory.ini
-  ```
 
 * Export Variables:
 

docs/cloud_providers/openstack.md

@@ -1,3 +1,4 @@
+
 # OpenStack
 
 ## Known compatible public clouds

docs/operations/recover-control-plane.md

@@ -1,3 +1,4 @@
+
 # Recovering the control plane
 
 To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook.
@@ -7,6 +8,7 @@ Examples of what broken means in this context:
 * One or more bare metal node(s) suffer from unrecoverable hardware failure
 * One or more node(s) fail during patching or upgrading
 * Etcd database corruption
+
 * Other node related failures leaving your control plane degraded or nonfunctional
 
 __Note that you need at least one functional node to be able to recover using this method.__

tests/scripts/check_typo.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# cd to the root directory of kubespray
+cd $(dirname $0)/../../
+
+rm ./misspell*
+
+set -e
+wget https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz
+tar -zxvf ./misspell_0.3.4_linux_64bit.tar.gz
+chmod 755 ./misspell
+git ls-files | grep -v OWNERS_ALIASES | xargs ./misspell -error

tests/scripts/collection-build-install.sh

@@ -1,7 +0,0 @@
-#!/bin/sh -e
-export ANSIBLE_COLLECTIONS_PATH="./ansible_collections"
-ansible-galaxy collection build --force
-ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
-ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
-test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
-test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml

tests/scripts/md-table/main.py

@@ -4,6 +4,7 @@ import sys
 import glob
 from pathlib import Path
 import yaml
+from pydblite import Base
 import re
 import jinja2
 import sys
@@ -13,7 +14,6 @@ from pprint import pprint
 
 parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
 parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
-parser.add_argument('--output', default='docs/developers/ci.md', help='output file')
 
 
 args = parser.parse_args()
@@ -24,26 +24,25 @@ env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
 # Data represents CI coverage data matrix
 class Data:
     def __init__(self):
-        self.container_managers = set()
+        self.db = Base(':memory:')
-        self.network_plugins = set()
+        self.db.create('container_manager', 'network_plugin', 'operating_system')
-        self.os = set()
-        self.combination = set()
 
 
-    def set(self, container_manager, network_plugin, os):
+    def set(self, container_manager, network_plugin, operating_system):
-        self.container_managers.add(container_manager)
+        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-        self.network_plugins.add(network_plugin)
+        self.db.commit()
-        self.os.add(os)
+    def exists(self, container_manager, network_plugin, operating_system):
-        self.combination.add(container_manager+network_plugin+os)
+        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
-
-    def exists(self, container_manager, network_plugin, os):
-        return (container_manager+network_plugin+os) in self.combination
 
     def jinja(self):
         template = env.get_template('table.md.j2')
-        container_engines = sorted(self.container_managers)
+        container_engines = list(self.db.get_unique_ids('container_manager'))
-        network_plugins = sorted(self.network_plugins)
+        network_plugins = list(self.db.get_unique_ids("network_plugin"))
-        operating_systems = sorted(self.os)
+        operating_systems = list(self.db.get_unique_ids("operating_system"))
+
+        container_engines.sort()
+        network_plugins.sort()
+        operating_systems.sort()
 
         return template.render(
             container_engines=container_engines,
@@ -92,5 +91,6 @@ for f in files:
     network_plugin = y.get('kube_network_plugin', 'calico')
     x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
     operating_system = x.group(1)
-    data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)
+    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-print(data.jinja(), file=open(args.output, 'w'))
+#print(data.markdown())
+print(data.jinja())

tests/scripts/md-table/requirements.txt

@@ -0,0 +1,4 @@
+jinja2
+pathlib ; python_version < '3.10'
+pyaml
+pydblite

tests/scripts/md-table/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -euxo pipefail
+
+echo "Install requirements..."
+pip install -r ./tests/scripts/md-table/requirements.txt
+
+echo "Generate current file..."
+./tests/scripts/md-table/main.py > tmp.md
+
+echo "Compare docs/developers/ci.md with actual tests in tests/files/*.yml ..."
+cmp docs/developers/ci.md tmp.md