Skip to content

Commit

Permalink
🍱 [patch] Wait for pre-hook jobs finished before continuing (#132)
Browse files Browse the repository at this point in the history
* 🍱 Wait for pre-hook finished before continuing

* ✅ Add unit tests

* 🔧 Increase helm validation timeout
  • Loading branch information
Pohfy123 authored May 13, 2022
1 parent 0f5bfaa commit 4e8b370
Show file tree
Hide file tree
Showing 7 changed files with 185 additions and 36 deletions.
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ require (
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/spec v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gofrs/uuid v3.3.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,6 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8=
Expand Down Expand Up @@ -657,7 +656,6 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
Expand Down
6 changes: 5 additions & 1 deletion internal/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"time"

"helm.sh/helm/v3/pkg/release"
"sigs.k8s.io/controller-runtime/pkg/client"

s2hv1 "github.com/agoda-com/samsahai/api/v1"
)
Expand All @@ -30,12 +31,15 @@ type DeployEngine interface {
// ForceDelete deletes environment when timeout
ForceDelete(refName string) error

// GetLabelSelector returns map of label for select the components that created by the engine
// GetLabelSelectors returns map of label for select the components that created by the engine
GetLabelSelectors(refName string) map[string]string

// GetReleases returns all deployed releases
GetReleases() ([]*release.Release, error)

// WaitForPreHookReady waits until all pre-hook pods are completed
WaitForPreHookReady(k8sClient client.Client, refName string) (bool, error)

// IsMocked uses for skip some functions due to mock deploy
//
// Skipped function: WaitForComponentsCleaned
Expand Down
21 changes: 18 additions & 3 deletions internal/staging/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,21 @@ func (c *controller) deployEnvironment(queue *s2hv1.Queue) error {
return nil
}

// verifies pre-hooks are completed
for _, rel := range releases {
isCompleted, err := deployEngine.WaitForPreHookReady(c.client, rel.Name)
if err != nil {
logger.Error(err, "error occurs while waiting for pre-hook ready",
"release", rel.Name, "queue", queue.Name)
return err
}

if !isCompleted {
time.Sleep(2 * time.Second)
return nil
}
}

if len(releases) != 0 && queue.IsPullRequestQueue() {
if err := c.deployActiveServicesIntoPullRequestEnvironment(); err != nil {
logger.Error(err, "cannot deploy active services into pull request environment",
Expand Down Expand Up @@ -436,8 +451,8 @@ func (c *controller) deployComponents(
releaseRevision[rel.Name] = rel.Version
}

timeout := 5 * time.Minute
ctx, cancelFunc := context.WithTimeout(context.Background(), timeout)
helmValidationTimeout := 15 * time.Minute
ctx, cancelFunc := context.WithTimeout(context.Background(), helmValidationTimeout)
defer cancelFunc()

isDeployedCh := make(chan bool, 2)
Expand Down Expand Up @@ -477,7 +492,7 @@ func (c *controller) deployComponents(
for i := 0; i < 2; i++ {
select {
case <-ctx.Done():
logger.Warnf("validating helm release took longer than %.0f seconds, queue: %s", timeout.Seconds(),
logger.Warnf("validating helm release took longer than %.0f seconds, queue: %s", helmValidationTimeout.Seconds(),
queue.Name)

var postInstalledReleases []*release.Release
Expand Down
103 changes: 74 additions & 29 deletions internal/staging/deploy/helm3/engine.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package helm3

import (
"context"
"encoding/json"
"fmt"
"os"
Expand All @@ -18,6 +19,9 @@ import (
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/release"
"helm.sh/helm/v3/pkg/storage/driver"
batchv1 "k8s.io/api/batch/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"

s2hv1 "github.com/agoda-com/samsahai/api/v1"
"github.com/agoda-com/samsahai/internal"
Expand Down Expand Up @@ -190,13 +194,19 @@ func (e *engine) GetReleases() ([]*release.Release, error) {
return releases, nil
}

func (e *engine) WaitForPreHookReady(k8sClient client.Client, refName string) (bool, error) {
selectors := e.GetLabelSelectors(refName)
listOpt := &client.ListOptions{Namespace: e.namespace, LabelSelector: labels.SelectorFromSet(selectors)}
return e.isPreHookJobsReady(k8sClient, listOpt)
}

func (e *engine) helmUninstall(refName string, disableHooks bool) error {
client := action.NewUninstall(e.actionSettings)
client.Timeout = DefaultUninstallTimeout
client.DisableHooks = disableHooks
helmCli := action.NewUninstall(e.actionSettings)
helmCli.Timeout = DefaultUninstallTimeout
helmCli.DisableHooks = disableHooks

logger.Debug("deleting release", "refName", refName)
if _, err := client.Run(refName); err != nil {
if _, err := helmCli.Run(refName); err != nil {
switch {
case errors.Is(errors.Cause(err), driver.ErrReleaseNotFound): // nolint
return nil
Expand Down Expand Up @@ -235,14 +245,14 @@ func (e *engine) helmInstall(
) error {
logger.Debug("helm install", "releaseName", refName, "chartName", chartName)

client := action.NewInstall(e.actionSettings)
client.ChartPathOptions = cpo
client.Namespace = e.namespace
client.ReleaseName = refName
client.DisableOpenAPIValidation = true
helmCli := action.NewInstall(e.actionSettings)
helmCli.ChartPathOptions = cpo
helmCli.Namespace = e.namespace
helmCli.ReleaseName = refName
helmCli.DisableOpenAPIValidation = true
if deployTimeout != nil {
client.Timeout = *deployTimeout
client.Wait = true
helmCli.Timeout = *deployTimeout
helmCli.Wait = true
}

ch, err := e.helmPrepareChart(chartName, cpo)
Expand All @@ -252,7 +262,7 @@ func (e *engine) helmInstall(
}
logger.Debug("helm prepare chart", "releaseName", refName, "chartName", chartName)

_, err = client.Run(ch, values)
_, err = helmCli.Run(ch, values)
if err != nil {
logger.Error(err, "helm install failed", "releaseName", refName, "chartName", chartName)
return errors.Wrapf(err, "helm install failed")
Expand All @@ -271,14 +281,14 @@ func (e *engine) helmUpgrade(
) error {
logger.Debug("helm upgrade", "releaseName", refName, "chartName", chartName)

client := action.NewUpgrade(e.actionSettings)
client.ChartPathOptions = cpo
client.Namespace = e.namespace
client.Atomic = true
client.DisableOpenAPIValidation = true
helmCli := action.NewUpgrade(e.actionSettings)
helmCli.ChartPathOptions = cpo
helmCli.Namespace = e.namespace
helmCli.Atomic = true
helmCli.DisableOpenAPIValidation = true
if deployTimeout != nil {
client.Timeout = *deployTimeout
client.Wait = true
helmCli.Timeout = *deployTimeout
helmCli.Wait = true
}

ch, err := e.helmPrepareChart(chartName, cpo)
Expand All @@ -287,7 +297,7 @@ func (e *engine) helmUpgrade(
return err
}

_, err = client.Run(refName, ch, values)
_, err = helmCli.Run(refName, ch, values)
if err != nil {
logger.Error(err, "helm upgrade failed", "releaseName", refName, "chartName", chartName)
return errors.Wrapf(err, "helm upgrade failed")
Expand All @@ -299,10 +309,10 @@ func (e *engine) helmUpgrade(
func (e *engine) helmRollback(refName string, revision int) error {
logger.Debug("helm rollback", "releaseName", refName, "revision", revision)

client := action.NewRollback(e.actionSettings)
client.Version = revision
helmCli := action.NewRollback(e.actionSettings)
helmCli.Version = revision

err := client.Run(refName)
err := helmCli.Run(refName)
if err != nil {
logger.Error(err, "helm rollback failed", "releaseName", refName, "revision", revision)
return errors.Wrapf(err, "helm rollback failed")
Expand Down Expand Up @@ -362,10 +372,10 @@ func (e *engine) helmPrepareChart(
}

func (e *engine) helmList() ([]*release.Release, error) {
client := action.NewList(e.actionSettings)
client.StateMask = action.ListAll
client.All = true
releases, err := client.Run()
helmCli := action.NewList(e.actionSettings)
helmCli.StateMask = action.ListAll
helmCli.All = true
releases, err := helmCli.Run()
if err != nil {
return nil, err
}
Expand All @@ -380,8 +390,8 @@ func (e *engine) helmGetValues() (map[string][]byte, error) {

valuesYaml := make(map[string][]byte)
for _, r := range releases {
client := action.NewGetValues(e.actionSettings)
values, err := client.Run(r.Name)
helmCli := action.NewGetValues(e.actionSettings)
values, err := helmCli.Run(r.Name)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -417,6 +427,41 @@ func (e *engine) ensureReleaseName(refName string) (string, error) {
return "", nil
}

func (e *engine) isPreHookJobsReady(k8sClient client.Client, listOpt *client.ListOptions) (bool, error) {
jobs := &batchv1.JobList{}

err := k8sClient.List(context.TODO(), jobs, listOpt)
if err != nil {
logger.Error(err, "list jobs error", "namespace", e.namespace)
return false, err
}

isReady := isPreHookJobsReady(jobs)
return isReady, nil
}

func isPreHookJobsReady(jobs *batchv1.JobList) bool {
for _, job := range jobs.Items {
for annotationKey, annotationVals := range job.Annotations {
if annotationKey == release.HookAnnotation {
hookEvents := strings.Split(annotationVals, ",")
for _, hookEvent := range hookEvents {
if hookEvent == string(release.HookPreInstall) || hookEvent == string(release.HookPreUpgrade) {
if job.Status.CompletionTime == nil {
return false
}
break
}
}

break
}
}
}

return true
}

// DeleteAllReleases deletes all releases in the namespace
func DeleteAllReleases(ns string, debug bool) error {
e := New(ns, debug).(*engine)
Expand Down
83 changes: 83 additions & 0 deletions internal/staging/deploy/helm3/engine_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package helm3

import (
"testing"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"helm.sh/helm/v3/pkg/release"
batchv1 "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/agoda-com/samsahai/internal/util/unittest"
)

func TestHelm3Engine(t *testing.T) {
unittest.InitGinkgo(t, "Helm3 Engine")
}

var _ = Describe("Helm3 Engine", func() {
g := NewWithT(GinkgoT())
mockNamespace := "test"
timeNow := metav1.Now()

Describe("pre-hook jobs", func() {

It("should successfully check pre-hooks job - all are ready", func() {
mockJobs := batchv1.JobList{
Items: []batchv1.Job{
{
ObjectMeta: metav1.ObjectMeta{
Name: "pre-hook-job-1",
Namespace: mockNamespace,
Annotations: map[string]string{
release.HookAnnotation: "pre-install,pre-upgrade",
},
},
Status: batchv1.JobStatus{
CompletionTime: &timeNow,
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "pre-hook-job-2",
Namespace: mockNamespace,
Annotations: map[string]string{
release.HookAnnotation: "pre-install",
},
},
Status: batchv1.JobStatus{
CompletionTime: &timeNow,
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "job-1",
Namespace: mockNamespace,
},
},
},
}

isReady := isPreHookJobsReady(&mockJobs)
g.Expect(isReady).To(BeTrue(), "all pre-hook jobs are completed")
})

It("should successfully check pre-hooks job - no pre-hook jobs", func() {
mockJobs := batchv1.JobList{
Items: []batchv1.Job{
{
ObjectMeta: metav1.ObjectMeta{
Name: "job-1",
Namespace: mockNamespace,
},
},
},
}

isReady := isPreHookJobsReady(&mockJobs)
g.Expect(isReady).To(BeTrue(), "all pre-hook jobs are completed")
})

})
})
5 changes: 5 additions & 0 deletions internal/staging/deploy/mock/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"time"

"helm.sh/helm/v3/pkg/release"
"sigs.k8s.io/controller-runtime/pkg/client"

s2hv1 "github.com/agoda-com/samsahai/api/v1"
"github.com/agoda-com/samsahai/internal"
Expand Down Expand Up @@ -93,6 +94,10 @@ func (e *engine) GetLabelSelectors(refName string) map[string]string {
return nil
}

func (e *engine) WaitForPreHookReady(k8sClient client.Client, refName string) (bool, error) {
return true, nil
}

func (e *engine) IsMocked() bool {
return true
}

0 comments on commit 4e8b370

Please sign in to comment.