Handle tombstones that were written before kubexit started.

.github/workflows/e2e-test.yaml

@@ -10,6 +10,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - name: Install Go
+        # https://github.com/mvdan/github-actions-golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.15.x"
+      - name: Install goimports
+        run: |
+          GO111MODULE=off go get golang.org/x/tools/cmd/goimports
+          echo "::add-path::$(go env GOPATH)/bin"
       - name: Checkout
         uses: actions/checkout@v2
       - name: Lint Go Code
@@ -25,6 +34,7 @@ jobs:
         uses: docker/build-push-action@v1
         with:
           repository: karlkfi/kubexit
+          tag_with_sha: true
           # Docker Automated Builds handle publishing
           push: false
       - name: Setup kube tools
@@ -53,12 +63,26 @@ jobs:
           kubectl get pods -n kube-system
           echo "current-context:" "$(kubectl config current-context)"
           echo "environment-kubeconfig:" "${KUBECONFIG}"
-      - name: Apply job
-        run: ci/e2e-test/client-server/apply-job.sh
-      - name: Await job
-        run: ci/e2e-test/client-server/await-job.sh
-      - name: Delete job
-        run: ci/e2e-test/client-server/delete-job.sh
+      - name: Load kubexit Image
+        run: kind load docker-image "karlkfi/kubexit:sha-${GITHUB_SHA:0:7}" --name e2e-test
+      - name: Build & Load test-client Image
+        run: |
+          set -o errexit -o nounset -o pipefail
+          IMAGE_TAG="karlkfi/test-client:sha-${GITHUB_SHA:0:7}"
+          docker build --tag "${IMAGE_TAG}" ci/e2e-test/assets/test-client/
+          kind load docker-image "${IMAGE_TAG}" --name e2e-test
+      - name: Test client-server
+        run: |
+          set -o errexit -o nounset -o pipefail
+          ci/e2e-test/client-server/apply-job.sh
+          ci/e2e-test/client-server/await-job.sh
+          ci/e2e-test/client-server/delete-job.sh
+      - name: Test client-server-slow-start
+        run: |
+          set -o errexit -o nounset -o pipefail
+          ci/e2e-test/client-server-slow-start/apply-job.sh
+          ci/e2e-test/client-server-slow-start/await-job.sh
+          ci/e2e-test/client-server-slow-start/delete-job.sh
       - name: Delete kind cluster
         if: always()
         run: kind delete cluster --name e2e-test

ci/e2e-test/assets/test-client/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:3.11
+RUN apk --no-cache add ca-certificates tzdata curl bash
+ENTRYPOINT ["curl"]

ci/e2e-test/client-server-slow-start/apply-job.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail -o posix
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+IMAGE_TAG=latest
+if [[ -n "${GITHUB_SHA:-}" ]]; then
+  IMAGE_TAG="sha-${GITHUB_SHA:0:7}"
+fi
+
+echo "Image: karlkfi/kubexit:${IMAGE_TAG}"
+kustomize edit set image "karlkfi/kubexit=karlkfi/kubexit:${IMAGE_TAG}"
+kustomize edit set image "karlkfi/test-client=karlkfi/test-client:${IMAGE_TAG}"
+kustomize build . | kubectl apply -f -

ci/e2e-test/client-server-slow-start/await-job.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail -o posix
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
+cd "${REPO_ROOT}"
+
+echo "Awaiting Job Init..."
+kubectl wait pod --selector=job-name=client-server-job --for=condition=Initialized --timeout=2m
+
+echo
+echo "Awaiting Job Completed or Error..."
+timeout=120
+SECONDS=0
+while (( SECONDS < timeout )); do
+    job_status="$(kubectl get pods --selector=job-name=client-server-job -o jsonpath="{.items[*].status.containerStatuses[*].state.terminated.reason}")"
+    if [[ "${job_status}" == "Completed Completed" || "${job_status}" == *"Error" ]]; then
+        echo "Status: ${job_status}"
+        break
+    fi
+    echo "Sleeping 2s..."
+    sleep 2
+done
+
+echo
+echo "Container Logs:"
+kubectl logs --selector=job-name=client-server-job --all-containers --tail=-1
+
+echo
+echo "Pod Respurces:"
+kubectl get pods --selector=job-name=client-server-job -o json | jq '.items[].status'
+
+echo "Status: ${job_status}"
+if [[ "${job_status}" != *"Error" ]]; then
+    echo "Expected: Error"
+    exit 1
+fi

ci/e2e-test/client-server-slow-start/delete-job.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail -o posix
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
+cd "${REPO_ROOT}"
+
+kubectl delete job client-server-job
+
+echo
+echo "Awaiting Pod Deletion..."
+kubectl wait pods --for=delete --selector=job-name=client-server-job --timeout=2m

ci/e2e-test/client-server-slow-start/job.yaml

@@ -0,0 +1,94 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: client-server-job
+spec:
+  backoffLimit: 1
+  template:
+    metadata:
+      labels:
+        app: client-server-job
+    spec:
+      serviceAccountName: client-server-job
+      restartPolicy: Never
+      volumes:
+      - name: graveyard
+        emptyDir:
+          medium: Memory
+      - name: kubexit
+        emptyDir: {}
+
+      initContainers:
+      - name: kubexit
+        image: karlkfi/kubexit:latest
+        command: ['cp', '/bin/kubexit', '/kubexit/kubexit']
+        volumeMounts:
+        - mountPath: /kubexit
+          name: kubexit
+
+      containers:
+      - name: client
+        image: karlkfi/test-client:latest
+        command: ['sh', '-c']
+        args:
+        - |
+          set -o errexit -o nounset -o pipefail
+          /kubexit/kubexit curl -v --fail http://localhost:80/
+        env:
+        - name: KUBEXIT_NAME
+          value: client
+        - name: KUBEXIT_GRAVEYARD
+          value: /graveyard
+        - name: KUBEXIT_BIRTH_DEPS
+          value: server
+        - name: KUBEXIT_BIRTH_TIMEOUT
+          value: "15s"
+        - name: KUBEXIT_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: KUBEXIT_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - mountPath: /graveyard
+          name: graveyard
+        - mountPath: /kubexit
+          name: kubexit
+      - name: server
+        image: nginx:1.17-alpine
+        command: ['sh', '-c']
+        args:
+        - |
+          set -o errexit -o nounset -o pipefail
+          sleep 30
+          /kubexit/kubexit nginx -g 'daemon off;'
+        env:
+        - name: KUBEXIT_NAME
+          value: server
+        - name: KUBEXIT_GRAVEYARD
+          value: /graveyard
+        - name: KUBEXIT_DEATH_DEPS
+          value: client
+        volumeMounts:
+        - mountPath: /graveyard
+          name: graveyard
+        - mountPath: /kubexit
+          name: kubexit
+        livenessProbe:
+          tcpSocket:
+            port: 80
+          initialDelaySeconds: 30
+          periodSeconds: 5
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 6
+        readinessProbe:
+          tcpSocket:
+            port: 80
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 6

ci/e2e-test/client-server-slow-start/kustomization.yaml

@@ -0,0 +1,14 @@
+resources:
+- service-account.yaml
+- role.yaml
+- role-binding.yaml
+- job.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: karlkfi/kubexit
+  newName: karlkfi/kubexit
+  newTag: latest
+- name: karlkfi/test-client
+  newName: karlkfi/test-client
+  newTag: latest

ci/e2e-test/client-server-slow-start/role-binding.yaml

@@ -0,0 +1,13 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: namespace-viewer
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: client-server-job
+  apiGroup: ""
+roleRef:
+  kind: Role
+  name: namespace-viewer
+  apiGroup: rbac.authorization.k8s.io

ci/e2e-test/client-server-slow-start/role.yaml

@@ -0,0 +1,9 @@
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: namespace-viewer
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]

ci/e2e-test/client-server-slow-start/service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: client-server-job
+  namespace: default

ci/e2e-test/client-server/apply-job.sh

@@ -4,5 +4,12 @@ set -o errexit -o nounset -o pipefail -o posix
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
 
-kustomize edit set image karlkfi/kubexit=karlkfi/kubexit:latest
+IMAGE_TAG=latest
+if [[ -n "${GITHUB_SHA:-}" ]]; then
+  IMAGE_TAG="sha-${GITHUB_SHA:0:7}"
+fi
+
+echo "Image: karlkfi/kubexit:${IMAGE_TAG}"
+kustomize edit set image "karlkfi/kubexit=karlkfi/kubexit:${IMAGE_TAG}"
+kustomize edit set image "karlkfi/test-client=karlkfi/test-client:${IMAGE_TAG}"
 kustomize build . | kubectl apply -f -

ci/e2e-test/client-server/await-job.sh

@@ -32,5 +32,6 @@ kubectl get pods --selector=job-name=client-server-job -o json | jq '.items[].st
 
 echo "Status: ${job_status}"
 if [[ "${job_status}" == *"Error" ]]; then
+    echo "Expected: Completed Completed"
     exit 1
 fi

ci/e2e-test/client-server/job.yaml

@@ -28,12 +28,11 @@ spec:
 
       containers:
       - name: client
-        image: alpine:3.11
+        image: karlkfi/test-client:latest
         command: ['sh', '-c']
         args:
         - |
           set -o errexit -o nounset -o pipefail
-          apk --no-cache add ca-certificates tzdata curl bash
           /kubexit/kubexit curl -v --fail http://localhost:80/
         env:
         - name: KUBEXIT_NAME

ci/e2e-test/client-server/kustomization.yaml

@@ -9,3 +9,6 @@ images:
 - name: karlkfi/kubexit
   newName: karlkfi/kubexit
   newTag: latest
+- name: karlkfi/test-client
+  newName: karlkfi/test-client
+  newTag: latest