CI: reduce VM resources requests to improve scheduling

Cleanup galaxy.yml

.gitlab-ci.yml

@@ -77,7 +77,6 @@ ci-authorized:
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml
-  - .gitlab-ci/shellcheck.yml
   - .gitlab-ci/terraform.yml
   - .gitlab-ci/packet.yml
   - .gitlab-ci/vagrant.yml

.gitlab-ci/lint.yml

@@ -1,13 +1,24 @@
 ---
-yamllint:
+generate-pre-commit:
-  extends: .job
+  image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
-  stage: unit-tests
+  stage: build
-  tags: [light]
+  before_script: []
-  variables:
-    LANG: C.UTF-8
   script:
-    - yamllint --strict .
+    - >
-  except: ['triggers', 'master']
+      yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
+      sed 's/^/      - /' |
+      cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
+  artifacts:
+    paths:
+      - pre-commit-generated.yml
+
+run-pre-commit:
+  stage: unit-tests
+  trigger:
+    include:
+      - artifact: pre-commit-generated.yml
+        job: generate-pre-commit
+    strategy: depend
 
 vagrant-validate:
   extends: .job
@@ -19,108 +30,11 @@ vagrant-validate:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
 
-ansible-lint:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - ansible-lint -v
-  except: ['triggers', 'master']
-
-jinja-syntax-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
-  except: ['triggers', 'master']
-
-syntax-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    ANSIBLE_INVENTORY: inventory/local-tests.cfg
-    ANSIBLE_REMOTE_USER: root
-    ANSIBLE_BECOME: "true"
-    ANSIBLE_BECOME_USER: root
-    ANSIBLE_VERBOSITY: "3"
-  script:
-    - ansible-playbook --syntax-check cluster.yml
-    - ansible-playbook --syntax-check playbooks/cluster.yml
-    - ansible-playbook --syntax-check upgrade-cluster.yml
-    - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
-    - ansible-playbook --syntax-check reset.yml
-    - ansible-playbook --syntax-check playbooks/reset.yml
-    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
-  except: ['triggers', 'master']
-
-collection-build-install-sanity-check:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
-  script:
-    - ansible-galaxy collection build
-    - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
-    - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
-    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
-    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
-  except: ['triggers', 'master']
-
-tox-inventory-builder:
-  stage: unit-tests
-  tags: [light]
-  extends: .job
-  before_script:
-    - ./tests/scripts/rebase.sh
-  script:
-    - pip3 install tox
-    - cd contrib/inventory_builder && tox
-  except: ['triggers', 'master']
-
-markdownlint:
-  stage: unit-tests
-  tags: [light]
-  image: node
-  before_script:
-    - npm install -g markdownlint-cli@0.22.0
-  script:
-    - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
-
-generate-sidebar:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  script:
-    - scripts/gen_docs_sidebar.sh
-    - git diff --exit-code
-
-check-readme-versions:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/check_readme_versions.sh
 
+# TODO: convert to pre-commit hook
 check-galaxy-version:
   stage: unit-tests
   tags: [light]
   image: python:3
   script:
     - tests/scripts/check_galaxy_version.sh
-
-check-typo:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/check_typo.sh
-
-ci-matrix:
-  stage: unit-tests
-  tags: [light]
-  image: python:3
-  script:
-    - tests/scripts/md-table/test.sh

.gitlab-ci/pre-commit-dynamic-stub.yml

@@ -0,0 +1,17 @@
+---
+# stub pipeline for dynamic generation
+pre-commit:
+  tags:
+  - light
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  variables:
+    PRE_COMMIT_HOME: /pre-commit-cache
+  script:
+  - pre-commit run -a $HOOK_ID
+  cache:
+    key: pre-commit-$HOOK_ID
+    paths:
+    - /pre-commit-cache
+  parallel:
+    matrix:
+    - HOOK_ID:

.gitlab-ci/shellcheck.yml

@@ -1,16 +0,0 @@
----
-shellcheck:
-  extends: .job
-  stage: unit-tests
-  tags: [light]
-  variables:
-    SHELLCHECK_VERSION: v0.7.1
-  before_script:
-    - ./tests/scripts/rebase.sh
-    - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
-    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
-    - shellcheck --version
-  script:
-    # Run shellcheck for all *.sh
-    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
-  except: ['triggers', 'master']

.markdownlint.yaml

@@ -1,3 +0,0 @@
----
-MD013: false
-MD029: false

.md_style.rb

@@ -0,0 +1,4 @@
+all
+exclude_rule 'MD013'
+exclude_rule 'MD029'
+rule 'MD007', :indent => 2

.mdlrc

@@ -0,0 +1 @@
+style "#{File.dirname(__FILE__)}/.md_style.rb"

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,47 +15,60 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.27.1
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/markdownlint/markdownlint
-    rev: v0.11.0
+    rev: v0.12.0
     hooks:
       - id: markdownlint
-        args: [-r, "~MD013,~MD029"]
+        exclude: "^.github|(^docs/_sidebar\\.md$)"
-        exclude: "^.git"
 
-  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: 3.0.0
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
-        args: [--severity, "error"]
+        args: ["--severity=error"]
         exclude: "^.git"
         files: "\\.sh$"
 
-  - repo: local
+  - repo: https://github.com/ansible/ansible-lint
+    rev: v24.5.0
     hooks:
       - id: ansible-lint
-        name: ansible-lint
-        entry: ansible-lint -v
-        language: python
-        pass_filenames: false
         additional_dependencies:
-          - .[community]
+          - ansible==9.5.1
+          - jsonschema==4.22.0
+          - jmespath==1.0.1
+          - netaddr==1.2.1
+          - distlib
 
+  - repo: https://github.com/VannTen/misspell
+    # Waiting on https://github.com/golangci/misspell/pull/19 to get merged
+    rev: 8592a4e
+    hooks:
+      - id: misspell
+        exclude: "OWNERS_ALIASES$"
+
+  - repo: local
+    hooks:
       - id: ansible-syntax-check
         name: ansible-syntax-check
         entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
         language: python
         files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
+        additional_dependencies:
+          - ansible==9.5.1
 
       - id: tox-inventory-builder
         name: tox-inventory-builder
         entry: bash -c "cd contrib/inventory_builder && tox"
         language: python
         pass_filenames: false
+        additional_dependencies:
+          - tox==4.15.0
 
       - id: check-readme-versions
         name: check-readme-versions
@@ -63,6 +76,15 @@ repos:
         language: script
         pass_filenames: false
 
+      - id: collection-build-install
+        name: Build and install kubernetes-sigs.kubespray Ansible collection
+        language: python
+        additional_dependencies:
+          - ansible-core>=2.16.4
+          - distlib
+        entry: tests/scripts/collection-build-install.sh
+        pass_filenames: false
+
       - id: generate-docs-sidebar
         name: generate-docs-sidebar
         entry: scripts/gen_docs_sidebar.sh
@@ -71,9 +93,13 @@ repos:
 
       - id: ci-matrix
         name: ci-matrix
-        entry: tests/scripts/md-table/test.sh
+        entry: tests/scripts/md-table/main.py
-        language: script
+        language: python
         pass_filenames: false
+        additional_dependencies:
+          - jinja2
+          - pathlib
+          - pyaml
 
       - id: jinja-syntax-check
         name: jinja-syntax-check
@@ -82,4 +108,4 @@ repos:
         types:
           - jinja
         additional_dependencies:
-          - Jinja2
+          - jinja2

OWNERS_ALIASES

@@ -6,6 +6,7 @@ aliases:
     - mzaian
     - oomichi
     - yankay
+    - ant31
   kubespray-reviewers:
     - cyclinder
     - erikjiang
@@ -14,7 +15,6 @@ aliases:
     - vannten
     - yankay
   kubespray-emeritus_approvers:
-    - ant31
     - atoms
     - chadswen
     - luckysb

contrib/terraform/nifcloud/README.md

@@ -72,6 +72,7 @@ The setup looks like following
 
   ```bash
   ./generate-inventory.sh > sample-inventory/inventory.ini
+  ```
 
 * Export Variables:
 

docs/ansible/ansible.md

@@ -231,6 +231,7 @@ The following tags are defined in playbooks:
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
+| system-packages                | Install packages using OS package manager             |
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |

docs/cloud_providers/openstack.md

@@ -1,4 +1,3 @@
-
 # OpenStack
 
 ## Known compatible public clouds

docs/operations/offline-environment.md

@@ -103,7 +103,9 @@ If you use the settings like the one above, you'll need to define in your invent
   can store them anywhere as long as it's accessible by kubespray. It's recommended to use `*_version` in the path so
   that you don't need to modify this setting everytime kubespray upgrades one of these components.
 * `yum_repo`/`debian_repo`/`ubuntu_repo`: OS package repository depending on your OS, should point to your internal
-  repository. Adjust the path accordingly.
+  repository. Adjust the path accordingly. Used only for Docker/Containerd packages (if needed); other packages might
+  be installed from other repositories. You might disable installing packages from other repositories by skipping
+  the `system-packages` tag
 
 ## Install Kubespray Python Packages
 

docs/operations/recover-control-plane.md

@@ -1,4 +1,3 @@
-
 # Recovering the control plane
 
 To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook.
@@ -8,7 +7,6 @@ Examples of what broken means in this context:
 * One or more bare metal node(s) suffer from unrecoverable hardware failure
 * One or more node(s) fail during patching or upgrading
 * Etcd database corruption
-  
 * Other node related failures leaving your control plane degraded or nonfunctional
 
 __Note that you need at least one functional node to be able to recover using this method.__

galaxy.yml

@@ -9,42 +9,12 @@ authors:
 tags:
   - infrastructure
 repository: https://github.com/kubernetes-sigs/kubespray
+issues: https://github.com/kubernetes-sigs/kubespray/issues
+documentation: https://kubespray.io
 license_file: LICENSE
 dependencies:
   ansible.utils: '>=2.5.0'
   community.general: '>=3.0.0'
-build_ignore:
+manifest:
-  - .github
+  directives:
-  - '*.tar.gz'
+    - recursive-exclude tests **
-  - extra_playbooks
-  - inventory
-  - scripts
-  - test-infra
-  - .ansible-lint
-  - .editorconfig
-  - .gitignore
-  - .gitlab-ci
-  - .gitlab-ci.yml
-  - .gitmodules
-  - .markdownlint.yaml
-  - .nojekyll
-  - .pre-commit-config.yaml
-  - .yamllint
-  - Dockerfile
-  - FILES.json
-  - MANIFEST.json
-  - Makefile
-  - Vagrantfile
-  - _config.yml
-  - ansible.cfg
-  - requirements*txt
-  - setup.cfg
-  - setup.py
-  - index.html
-  - reset.yml
-  - cluster.yml
-  - scale.yml
-  - recover-control-plane.yml
-  - remove-node.yml
-  - upgrade-cluster.yml
-  - library

requirements.txt

@@ -1,10 +1,10 @@
-ansible==9.5.1
+ansible==9.6.0
 cryptography==42.0.7
 jinja2==3.1.4
 jmespath==1.0.1
+jsonschema==4.22.0
 MarkupSafe==2.1.5
 netaddr==1.2.1
 pbr==6.0.0
 ruamel.yaml==0.18.6
 ruamel.yaml.clib==0.2.8
-jsonschema==4.22.0

roles/container-engine/containerd/templates/config.toml.j2

@@ -107,4 +107,3 @@ oom_score = {{ containerd_oom_score }}
     sampling_ratio = {{ containerd_tracing_sampling_ratio }}
     service_name = "{{ containerd_tracing_service_name }}"
 {% endif %}
-

roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2

@@ -32,4 +32,3 @@ data:
       - name: helper-pod
         image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}"
         imagePullPolicy: IfNotPresent
-

roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2

@@ -11,4 +11,3 @@ subjects:
 - kind: ServiceAccount
   name: {{ node_feature_discovery_worker_sa_name }}
   namespace: {{ node_feature_discovery_namespace }}
-

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -1,10 +1,7 @@
 ---
 - name: Stop if either kube_control_plane or kube_node group is empty
   assert:
-    that: "groups.get( item )"
+    that: groups.get( 'kube_control_plane' )
-  with_items:
-    - kube_control_plane
-    - kube_node
   run_once: true
   when: not ignore_assert_errors
 

roles/kubernetes/preinstall/tasks/main.yml

@@ -68,6 +68,7 @@
     - not dns_late
   tags:
     - bootstrap-os
+    - system-packages
 
 - name: Apply system configurations
   import_tasks: 0080-system-configurations.yml

roles/network_plugin/calico/templates/calico-config.yml.j2

@@ -102,4 +102,3 @@ data:
         }
       ]
     }
-

roles/network_plugin/cilium/templates/hubble/service.yml.j2

@@ -102,4 +102,3 @@ spec:
     protocol: TCP
     targetPort: 4244
   internalTrafficPolicy: Local
-

tests/cloud_playbooks/roles/packet-ci/defaults/main.yml

@@ -4,15 +4,14 @@
 vm_cpu_cores: 2
 vm_cpu_sockets: 1
 vm_cpu_threads: 2
-vm_memory: 2048Mi
+vm_memory: 2048
 
 # Replace invalid characters so that we can use the branch name in kubernetes labels
 branch_name_sane: "{{ branch | regex_replace('/', '-') }}"
 
 # Request/Limit allocation settings
-
+cpu_allocation_ratio: 0.25
-cpu_allocation_ratio: 0.5
+memory_allocation_ratio: 0.5
-memory_allocation_ratio: 1
 
 # Default path for inventory
 inventory_path: "/tmp/{{ test_name }}/inventory"

tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2

@@ -4,6 +4,8 @@ kind: VirtualMachine
 metadata:
   name: "instance-{{ vm_id }}"
   namespace: "{{ test_name }}"
+  annotations:
+    kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2"
   labels:
     kubevirt.io/os: {{ cloud_image }}
 spec:
@@ -34,10 +36,10 @@ spec:
             threads: {{ vm_cpu_threads }}
         resources:
           requests:
-            memory: {{ vm_memory * memory_allocation_ratio }}
+            memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
             cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
           limits:
-            memory: {{ vm_memory }}
+            memory: "{{ vm_memory }}Mi"
             cpu: {{ vm_cpu_cores }}
       networks:
       - name: default

tests/files/packet_almalinux8-calico-ha-ebpf.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: ha
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 calico_bpf_enabled: true

tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 enable_nodelocaldns_secondary: true

tests/files/packet_almalinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_almalinux8-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Use docker
 container_manager: docker

tests/files/packet_almalinux8-kube-ovn.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: kube-ovn

tests/files/packet_rockylinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-cilium.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: cilium

tests/files/packet_ubuntu22-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu22-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-etcd-datastore.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: node-etcd-client
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/requirements.txt

@@ -5,8 +5,8 @@ ara[server]==1.7.1
 dopy==0.3.7
 molecule==24.2.1
 molecule-plugins[vagrant]==23.5.3
-python-vagrant==1.0.0
 pytest-testinfra==10.1.0
+python-vagrant==1.0.0
 tox==4.15.0
-yamllint==1.35.1
 tzdata==2024.1
+yamllint==1.35.1

tests/scripts/check_typo.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# cd to the root directory of kubespray
-cd $(dirname $0)/../../
-
-rm ./misspell*
-
-set -e
-wget https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz
-tar -zxvf ./misspell_0.3.4_linux_64bit.tar.gz
-chmod 755 ./misspell
-git ls-files | grep -v OWNERS_ALIASES | xargs ./misspell -error

tests/scripts/collection-build-install.sh

@@ -0,0 +1,7 @@
+#!/bin/sh -e
+export ANSIBLE_COLLECTIONS_PATH="./ansible_collections"
+ansible-galaxy collection build --force
+ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml

tests/scripts/md-table/main.py

@@ -4,7 +4,6 @@ import sys
 import glob
 from pathlib import Path
 import yaml
-from pydblite import Base
 import re
 import jinja2
 import sys
@@ -14,6 +13,7 @@ from pprint import pprint
 
 parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
 parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
+parser.add_argument('--output', default='docs/developers/ci.md', help='output file')
 
 
 args = parser.parse_args()
@@ -24,25 +24,26 @@ env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
 # Data represents CI coverage data matrix
 class Data:
     def __init__(self):
-        self.db = Base(':memory:')
+        self.container_managers = set()
-        self.db.create('container_manager', 'network_plugin', 'operating_system')
+        self.network_plugins = set()
+        self.os = set()
+        self.combination = set()
 
 
-    def set(self, container_manager, network_plugin, operating_system):
+    def set(self, container_manager, network_plugin, os):
-        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+        self.container_managers.add(container_manager)
-        self.db.commit()
+        self.network_plugins.add(network_plugin)
-    def exists(self, container_manager, network_plugin, operating_system):
+        self.os.add(os)
-        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
+        self.combination.add(container_manager+network_plugin+os)
+
+    def exists(self, container_manager, network_plugin, os):
+        return (container_manager+network_plugin+os) in self.combination
 
     def jinja(self):
         template = env.get_template('table.md.j2')
-        container_engines = list(self.db.get_unique_ids('container_manager'))
+        container_engines = sorted(self.container_managers)
-        network_plugins = list(self.db.get_unique_ids("network_plugin"))
+        network_plugins = sorted(self.network_plugins)
-        operating_systems = list(self.db.get_unique_ids("operating_system"))
+        operating_systems = sorted(self.os)
-
-        container_engines.sort()
-        network_plugins.sort()
-        operating_systems.sort()
 
         return template.render(
             container_engines=container_engines,
@@ -91,6 +92,5 @@ for f in files:
     network_plugin = y.get('kube_network_plugin', 'calico')
     x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
     operating_system = x.group(1)
-    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
+    data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)
-#print(data.markdown())
+print(data.jinja(), file=open(args.output, 'w'))
-print(data.jinja())

tests/scripts/md-table/requirements.txt

@@ -1,4 +0,0 @@
-jinja2
-pathlib ; python_version < '3.10'
-pyaml
-pydblite

tests/scripts/md-table/test.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-echo "Install requirements..."
-pip install -r ./tests/scripts/md-table/requirements.txt
-
-echo "Generate current file..."
-./tests/scripts/md-table/main.py > tmp.md
-
-echo "Compare docs/developers/ci.md with actual tests in tests/files/*.yml ..."
-cmp docs/developers/ci.md tmp.md