Add Configuration device plugin (#627) #1215
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test K3s, Kubernetes, and MicroK8s | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - test/run-end-to-end.py | |
| - test/run-conservation-of-broker-pod.py | |
| - test/run-helm-install-delete.py | |
| - test/run-webhook.py | |
| - test/shared_test_code.py | |
| - .github/workflows/run-test-cases.yml | |
| - build/containers/Dockerfile.agent | |
| - build/containers/Dockerfile.controller | |
| - deployment/helm/** | |
| - agent/** | |
| - controller/** | |
| - shared/** | |
| - version.txt | |
| - build/akri-containers.mk | |
| - Makefile | |
| push: | |
| branches: [main] | |
| paths: | |
| - test/run-end-to-end.py | |
| - test/run-conservation-of-broker-pod.py | |
| - test/run-helm-install-delete.py | |
| - test/run-webhook.py | |
| - test/shared_test_code.py | |
| - .github/workflows/run-test-cases.yml | |
| - build/containers/Dockerfile.agent | |
| - build/containers/Dockerfile.controller | |
| - deployment/helm/** | |
| - agent/** | |
| - controller/** | |
| - shared/** | |
| - version.txt | |
| - build/akri-containers.mk | |
| - Makefile | |
| release: | |
| types: | |
| - published | |
| jobs: | |
| build-containers: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout the head commit of the branch | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Build local containers for PR tests | |
| if: startsWith(github.event_name, 'pull_request') | |
| env: | |
| BUILD_AMD64: 1 | |
| BUILD_ARM32: 0 | |
| BUILD_ARM64: 0 | |
| BUILD_SLIM_AGENT: 0 | |
| AGENT_FEATURES: "agent-full" | |
| PACKAGES_TO_EXCLUDE: "akri-udev akri-onvif akri-opcua udev-video-broker debug-echo-discovery-handler onvif-discovery-handler opcua-discovery-handler udev-discovery-handler" | |
| PREFIX: ghcr.io/project-akri/akri | |
| LABEL_PREFIX: pr | |
| CARGO_INCREMENTAL: 0 | |
| run: | | |
| make akri-build | |
| make controller-build-amd64 | |
| make agent-full-build-amd64 | |
| make webhook-configuration-build-amd64 | |
| docker save ${PREFIX}/agent-full:${LABEL_PREFIX}-amd64 > agent.tar | |
| docker save ${PREFIX}/controller:${LABEL_PREFIX}-amd64 > controller.tar | |
| docker save ${PREFIX}/webhook-configuration:${LABEL_PREFIX}-amd64 > webhook-configuration.tar | |
| - name: Upload Agent container as artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: agent.tar | |
| path: agent.tar | |
| - name: Upload Controller container as artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: controller.tar | |
| path: controller.tar | |
| - name: Upload Webhook-Configuration container as artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: webhook-configuration.tar | |
| path: webhook-configuration.tar | |
| test-cases: | |
| needs: build-containers | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kube: | |
| - runtime: MicroK8s-1.24 | |
| version: 1.24/stable | |
| - runtime: MicroK8s-1.25 | |
| version: 1.25/stable | |
| - runtime: MicroK8s-1.26 | |
| version: 1.26/stable | |
| - runtime: MicroK8s-1.27 | |
| version: 1.27/stable | |
| - runtime: K3s-1.24 | |
| version: v1.24.13+k3s1 | |
| - runtime: K3s-1.25 | |
| version: v1.25.9+k3s1 | |
| - runtime: K3s-1.26 | |
| version: v1.26.4+k3s1 | |
| - runtime: K3s-1.27 | |
| version: v1.27.1+k3s1 | |
| - runtime: Kubernetes-1.24 | |
| version: 1.24.13-00 | |
| - runtime: Kubernetes-1.25 | |
| version: 1.25.9-00 | |
| - runtime: Kubernetes-1.26 | |
| version: 1.26.4-00 | |
| - runtime: Kubernetes-1.27 | |
| version: 1.27.1-00 | |
| test: | |
| - case: end-to-end | |
| file: test/run-end-to-end.py | |
| - case: webhook | |
| file: test/run-webhook.py | |
| steps: | |
| - name: Checkout the head commit of the branch | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Python kubernetes dependency | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install kubernetes | |
| - name: Download Agent container artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: agent.tar | |
| - name: Download Controller container artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: controller.tar | |
| - name: Download Webhook-Configuration container artifact | |
| if: startsWith(github.event_name, 'pull_request') | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: webhook-configuration.tar | |
| - if: startsWith(matrix.kube.runtime, 'K3s') | |
| name: Install K3s | |
| env: | |
| INSTALL_K3S_VERSION: ${{ matrix.kube.version }} | |
| run: | | |
| sudo curl -sfL https://get.k3s.io -o install.sh | |
| sudo chmod +x install.sh | |
| ./install.sh server --kubelet-arg=eviction-hard="imagefs.available<1%,nodefs.available<1%" --kubelet-arg=eviction-minimum-reclaim="imagefs.available=1%,nodefs.available=1%" | |
| mkdir -p $HOME/.kube | |
| sudo cp -i /etc/rancher/k3s/k3s.yaml $HOME/.kube/config | |
| sudo chown $(id -u):$(id -g) $HOME/.kube/config | |
| echo "--set kubernetesDistro=k3s" > /tmp/k8s_distro_to_test.txt | |
| echo 'kubectl' > /tmp/runtime_cmd_to_test.txt | |
| echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt | |
| until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for k3s to become ready"; sleep 10; done | |
| - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'K3s')) | |
| name: Import local agent and controller to K3s | |
| run: | | |
| sudo k3s ctr image import agent.tar | |
| sudo k3s ctr image import controller.tar | |
| sudo k3s ctr image import webhook-configuration.tar | |
| - if: startsWith(matrix.kube.runtime, 'Kubernetes') | |
| name: Install Kubernetes | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt-get install -y apt-transport-https ca-certificates curl | |
| sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list | |
| sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker.gpg | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list | |
| sudo apt-get update | |
| sudo apt-get install -o Dpkg::Options::="--force-overwrite" -y --allow-downgrades kubelet=${{ matrix.kube.version }} kubeadm=${{ matrix.kube.version }} kubectl=${{ matrix.kube.version }} containerd.io | |
| kubectl version && echo "kubectl return code: $?" || echo "kubectl return code: $?" | |
| kubeadm version && echo "kubeadm return code: $?" || echo "kubeadm return code: $?" | |
| kubelet --version && echo "kubelet return code: $?" || echo "kubelet return code: $?" | |
| cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf | |
| overlay | |
| br_netfilter | |
| EOF | |
| sudo modprobe overlay | |
| sudo modprobe br_netfilter | |
| cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf | |
| net.bridge.bridge-nf-call-iptables = 1 | |
| net.bridge.bridge-nf-call-ip6tables = 1 | |
| net.ipv4.ip_forward = 1 | |
| EOF | |
| sudo sysctl --system | |
| sudo mkdir -p /etc/containerd | |
| containerd config default | sudo tee /etc/containerd/config.toml | |
| sudo sed -i "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml | |
| sudo systemctl restart containerd | |
| sudo swapoff -a | |
| sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --cri-socket=unix:///run/containerd/containerd.sock | |
| mkdir -p $HOME/.kube | |
| sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config | |
| sudo chown $(id -u):$(id -g) $HOME/.kube/config | |
| kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml | |
| verlte() { [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] ; } | |
| verlt() { [ "$1" = "$2" ] && return 1 || verlte $1 $2 ; } | |
| # Kubernetes 1.24+ has control plane label | |
| if verlte "1.24.0" "${{ matrix.kube.version }}" ; then | |
| kubectl taint nodes --all node-role.kubernetes.io/control-plane- | |
| fi | |
| # Kubernetes before 1.25 has master label | |
| if verlt "${{ matrix.kube.version }}" "1.25.0" ; then | |
| kubectl taint nodes --all node-role.kubernetes.io/master- | |
| fi | |
| echo '--set kubernetesDistro=k8s' > /tmp/k8s_distro_to_test.txt | |
| echo 'kubectl' > /tmp/runtime_cmd_to_test.txt | |
| echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt | |
| until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for kubernetes to become ready"; sleep 10; done | |
| - if: startsWith(matrix.kube.runtime, 'Kubernetes') | |
| name: Output Kubelet, Containerd, Docker Logs | |
| run: | | |
| echo "Kubelet Logs:" | |
| sudo journalctl -xeu kubelet --no-pager | |
| echo "\nContainerd Logs" | |
| sudo journalctl -xeu containerd --no-pager | |
| echo "\nDocker Logs" | |
| sudo journalctl -xeu docker --no-pager | |
| - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'Kubernetes')) | |
| name: Import local agent and controller to Kubernetes | |
| run: | | |
| # Need to load to containerd with ctr | |
| # -n allows kubernetes to see the image | |
| sudo ctr -n=k8s.io image import agent.tar | |
| sudo ctr -n=k8s.io image import controller.tar | |
| sudo ctr -n=k8s.io image import webhook-configuration.tar | |
| sudo crictl --runtime-endpoint unix:///var/run/containerd/containerd.sock images | |
| - if: startsWith(matrix.kube.runtime, 'MicroK8s') | |
| name: Install MicroK8s | |
| run: | | |
| set -x | |
| sudo snap install microk8s --classic --channel=${{ matrix.kube.version }} | |
| sudo microk8s status --wait-ready | |
| sudo usermod -a -G microk8s $USER | |
| mkdir -p $HOME/.kube | |
| sudo microk8s.kubectl config view --raw > $HOME/.kube/config | |
| sudo chown $(id -u):$(id -g) $HOME/.kube/config | |
| sudo microk8s.enable rbac dns | |
| sudo sed -i 's/memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi/memory.available<25Mi,nodefs.available<50Mi,imagefs.available<50Mi/' /var/snap/microk8s/current/args/kubelet | |
| sudo systemctl restart snap.microk8s.daemon-kubelite | |
| until sudo microk8s.status --wait-ready; do sleep 5s; echo "Try again"; done | |
| echo '--set kubernetesDistro=microk8s' > /tmp/k8s_distro_to_test.txt | |
| echo 'kubectl' > /tmp/runtime_cmd_to_test.txt | |
| echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt | |
| - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'MicroK8s')) | |
| name: Import local agent and controller to MicroK8s | |
| run: | | |
| sudo microk8s.status --wait-ready | |
| until sudo microk8s ctr images ls; do sleep 5s; echo "Try again"; done | |
| sudo microk8s ctr images ls | |
| sudo microk8s ctr --debug --timeout 60s images import agent.tar | |
| sudo microk8s ctr --debug --timeout 60s images import controller.tar | |
| sudo microk8s ctr --debug --timeout 60s images import webhook-configuration.tar | |
| sudo microk8s ctr images ls | |
| - name: Add Akri Helm Chart | |
| run: helm repo add akri-helm-charts https://project-akri.github.io/akri/ | |
| # For push and release, we need to wait for the Helm chart and | |
| # associated containers to build. | |
| - if: github.event_name == 'push' || github.event_name == 'release' | |
| name: Set sleep duration before running script to 2700 | |
| run: echo 2700 > /tmp/sleep_duration.txt | |
| # For pull_request, use the locally built containers. | |
| - if: startsWith(github.event_name, 'pull_request') | |
| name: Tell Helm to use the 'local' labels for container images | |
| run: | | |
| git fetch origin main | |
| git show origin/main:version.txt > /tmp/version_to_test.txt | |
| echo '--set agent.image.pullPolicy=Never,agent.image.tag=pr-amd64,controller.image.pullPolicy=Never,controller.image.tag=pr-amd64,webhookConfiguration.image.pullPolicy=Never,webhookConfiguration.image.tag=pr-amd64' > /tmp/extra_helm_args.txt | |
| # For non-PR (i.e. push, release, manual), version.txt is corresponds | |
| # to an existing Helm chart. | |
| - if: (!(startsWith(github.event_name, 'pull_request'))) | |
| name: Use current version for push | |
| run: cat version.txt > /tmp/version_to_test.txt | |
| # For workflow_dispatch and pull_request, use the files in deployment/helm | |
| # as basis for helm install ... this enables us to test any changes made to | |
| # the helm chart files in a PR (where no helm chart is published) | |
| - if: github.event_name != 'push' && github.event_name != 'release' | |
| name: Tell Helm to use the files in deployment/helm to build chart | |
| run: | | |
| echo './deployment/helm' > /tmp/helm_chart_location.txt | |
| # For push, use a specific version of the `akri-dev` charts that are built and | |
| # published by the helm workflow. | |
| - if: github.event_name == 'push' | |
| name: Tell Helm to use the `akri-dev` published charts | |
| run: | | |
| echo "akri-helm-charts/akri-dev --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt | |
| # For release, use a specific version of the `akri` charts that are built and | |
| # published by the helm workflow. | |
| - if: github.event_name == 'release' | |
| name: Tell Helm to use the `akri` published charts | |
| run: | | |
| echo "akri-helm-charts/akri --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt | |
| - name: Execute test script ${{ matrix.test.file }} | |
| run: python ${{ matrix.test.file }} | |
| - name: Upload Agent log as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-agent-log | |
| path: /tmp/agent_log.txt | |
| - name: Upload controller log as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-controller-log | |
| path: /tmp/controller_log.txt | |
| - name: Upload webhook log as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-webhook-log | |
| path: /tmp/webhook_log.txt |