From 7a4ddb0779cc6437e3105799e6cd2c3e89f3b0f1 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 17 Jan 2020 15:15:47 -0800 Subject: [PATCH] Setup a prod environment for the issue label bot. * kubeflow/code-intelligence#70 * update create_secrets for prod. * Delete old K8s manifests its now replaced by kustomize packages. --- deployment/base/deployment.yaml | 2 +- deployment/certificate.mlbot-net.yaml | 19 ----- deployment/certificate.yaml | 19 ----- deployment/deployments-test.yaml | 79 ------------------- deployment/deployments.yaml | 77 ------------------ deployment/fake-secret.yaml | 13 --- deployment/ingress-test.yaml | 24 ------ deployment/ingress.mlbot-net.yaml | 23 ------ deployment/ingress.yaml | 24 ------ .../prod/certificate.yaml} | 5 +- deployment/overlays/prod/deployment.yaml | 6 +- deployment/overlays/prod/ingress.yaml | 7 ++ deployment/overlays/prod/kustomization.yaml | 15 ++++ deployment/service-test.yaml | 14 ---- deployment/service.yaml | 14 ---- script/create_secrets.py | 50 ++++++++---- script/send_request.py | 14 +++- 17 files changed, 77 insertions(+), 328 deletions(-) delete mode 100644 deployment/certificate.mlbot-net.yaml delete mode 100644 deployment/certificate.yaml delete mode 100644 deployment/deployments-test.yaml delete mode 100644 deployment/deployments.yaml delete mode 100644 deployment/fake-secret.yaml delete mode 100644 deployment/ingress-test.yaml delete mode 100644 deployment/ingress.mlbot-net.yaml delete mode 100644 deployment/ingress.yaml rename deployment/{certificate.mlbot-net-gke.yaml => overlays/prod/certificate.yaml} (62%) create mode 100644 deployment/overlays/prod/ingress.yaml create mode 100644 deployment/overlays/prod/kustomization.yaml delete mode 100644 deployment/service-test.yaml delete mode 100644 deployment/service.yaml diff --git a/deployment/base/deployment.yaml b/deployment/base/deployment.yaml index 07a0b2de..acb1d289 100644 --- a/deployment/base/deployment.yaml +++ b/deployment/base/deployment.yaml @@ -35,7 +35,7 @@ spec: valueFrom: secretKeyRef: name: ml-app-inference-secret - key: WEBHOOK_SECRET + key: WEBHOOK_SECRET # The values for the Kubeflow kf-label-bot-dev application # See kubeflow/code-intelligence#84. This is suitable # for development but shouldn't be used in production diff --git a/deployment/certificate.mlbot-net.yaml b/deployment/certificate.mlbot-net.yaml deleted file mode 100644 index e994b06a..00000000 --- a/deployment/certificate.mlbot-net.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: certmanager.k8s.io/v1alpha1 -kind: Certificate -metadata: - name: mlbot-net - namespace: mlapp -spec: - acme: - config: - - domains: - - mlbot.net - http01: - ingress: mlbot-net - commonName: mlbot.net - dnsNames: - - mlbot.net - issuerRef: - kind: ClusterIssuer - name: letsencrypt-prod - secretName: mlapp-tls diff --git a/deployment/certificate.yaml b/deployment/certificate.yaml deleted file mode 100644 index 9cc60c50..00000000 --- a/deployment/certificate.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: certmanager.k8s.io/v1alpha1 -kind: Certificate -metadata: - name: predict-mlapp-tls - namespace: mlapp -spec: - acme: - config: - - domains: - - predict.mlbot.net - http01: - ingress: ml-gh-app - commonName: predict.mlbot.net - dnsNames: - - predict.mlbot.net - issuerRef: - kind: ClusterIssuer - name: letsencrypt-prod - secretName: predict-mlapp-tls diff --git a/deployment/deployments-test.yaml b/deployment/deployments-test.yaml deleted file mode 100644 index 6eedcbc2..00000000 --- a/deployment/deployments-test.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ml-github-app-test - namespace: mlapp - labels: - app: ml-github-app -spec: - replicas: 1 - selector: - matchLabels: - app: ml-github-app - template: - metadata: - labels: - app: ml-github-app - spec: - containers: - - name: ml-github-app - image: gcr.io/issue-label-bot-dev/github-mlapp-test - command: ["python", "app.py"] - readinessProbe: - httpGet: - path: / - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 3 - env: - - name: PRIVATE_KEY - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: PRIVATE_KEY - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: DATABASE_URL - - name: WEBHOOK_SECRET - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: WEBHOOK_SECRET - - name: APP_ID - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: APP_ID - - name: GCP_PROJECT_ID - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: GCP_PROJECT_ID - - name: GCP_PUBSUB_TOPIC_NAME - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: GCP_PUBSUB_TOPIC_NAME - - name: PUBSUB_CREDENTIALS_JSON_BLOB - valueFrom: - secretKeyRef: - name: ml-app-inference-secret-test - key: PUBSUB_CREDENTIALS_JSON_BLOB - - name: FLASK_ENV - value: production - - name: PORT - value: '3000' - - name: APP_URL - value: https://mlbot.net/ - - name: APP_URL_HEROKU - value: https://fathomless-forest-27162.herokuapp.com/ - - name: authors - value: 'c' - - name: DEVELOPMENT_FLAG - value: 'True' - ports: - - containerPort: 443 - - containerPort: 80 - - containerPort: 3000 diff --git a/deployment/deployments.yaml b/deployment/deployments.yaml deleted file mode 100644 index 612f9967..00000000 --- a/deployment/deployments.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ml-github-app - namespace: mlapp - labels: - app: ml-github-app -spec: - replicas: 9 - selector: - matchLabels: - app: ml-github-app - template: - metadata: - labels: - app: ml-github-app - spec: - containers: - - name: ml-github-app - image: hamelsmu/mlapp - command: ["python", "app.py"] - readinessProbe: - httpGet: - path: / - port: 3000 - initialDelaySeconds: 10 - periodSeconds: 3 - env: - - name: PRIVATE_KEY - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: PRIVATE_KEY - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: DATABASE_URL - - name: WEBHOOK_SECRET - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: WEBHOOK_SECRET - - name: APP_ID - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: APP_ID - - name: GCP_PROJECT_ID - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: GCP_PROJECT_ID - - name: GCP_PUBSUB_TOPIC_NAME - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: GCP_PUBSUB_TOPIC_NAME - - name: PUBSUB_CREDENTIALS_JSON_BLOB - valueFrom: - secretKeyRef: - name: ml-app-inference-secret - key: PUBSUB_CREDENTIALS_JSON_BLOB - - name: FLASK_ENV - value: production - - name: PORT - value: '3000' - - name: APP_URL - value: https://mlbot.net/ - - name: APP_URL_HEROKU - value: https://fathomless-forest-27162.herokuapp.com/ - - name: authors - value: 'c' - ports: - - containerPort: 443 - - containerPort: 80 - - containerPort: 3000 diff --git a/deployment/fake-secret.yaml b/deployment/fake-secret.yaml deleted file mode 100644 index f6cc7337..00000000 --- a/deployment/fake-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret-manually -apiVersion: v1 -kind: Secret -metadata: - name: ml-app-inference-secret -type: Opaque -data: - PRIVATE_KEY: something - PRIVATE_KEY_DEV: something - DATABASE_URL: something - WEBHOOK_SECRET: something - APP_ID: something - APP_ID_DEV: something \ No newline at end of file diff --git a/deployment/ingress-test.yaml b/deployment/ingress-test.yaml deleted file mode 100644 index 59c1e92e..00000000 --- a/deployment/ingress-test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ml-gh-app-test - namespace: mlapp - annotations: - kubernetes.io/ingress.global-static-ip-name: mlbot-test - networking.gke.io/managed-certificates: mlbot-net-test -spec: - backend: - serviceName: ml-github-app-test - servicePort: 3000 - rules: - - http: - paths: - - path: / - backend: - serviceName: ml-github-app-test - servicePort: 3000 - # tls: - # - hosts: - # - predict.mlbot.net - # - mlbot.net - # secretName: predict-mlapp-tls diff --git a/deployment/ingress.mlbot-net.yaml b/deployment/ingress.mlbot-net.yaml deleted file mode 100644 index 4e5d1541..00000000 --- a/deployment/ingress.mlbot-net.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: mlbot-net - namespace: mlapp - annotations: - kubernetes.io/ingress.global-static-ip-name: mlbot-net - networking.gke.io/managed-certificates: mlbot-net-2 -spec: - backend: - serviceName: ml-github-app - servicePort: 3000 - rules: - - http: - paths: - - path: / - backend: - serviceName: ml-github-app - servicePort: 3000 - # tls: - # - hosts: - # - mlbot.net - # secretName: mlapp-tls \ No newline at end of file diff --git a/deployment/ingress.yaml b/deployment/ingress.yaml deleted file mode 100644 index 28e98341..00000000 --- a/deployment/ingress.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ml-gh-app - namespace: mlapp - annotations: - kubernetes.io/ingress.global-static-ip-name: mlbot - networking.gke.io/managed-certificates: mlbot-net-2 -spec: - backend: - serviceName: ml-github-app - servicePort: 3000 - rules: - - http: - paths: - - path: / - backend: - serviceName: ml-github-app - servicePort: 3000 - # tls: - # - hosts: - # - predict.mlbot.net - # - mlbot.net - # secretName: predict-mlapp-tls \ No newline at end of file diff --git a/deployment/certificate.mlbot-net-gke.yaml b/deployment/overlays/prod/certificate.yaml similarity index 62% rename from deployment/certificate.mlbot-net-gke.yaml rename to deployment/overlays/prod/certificate.yaml index ee361e09..e699fa8e 100644 --- a/deployment/certificate.mlbot-net-gke.yaml +++ b/deployment/overlays/prod/certificate.yaml @@ -1,8 +1,7 @@ apiVersion: networking.gke.io/v1beta1 kind: ManagedCertificate metadata: - name: mlbot-net-2 - namespace: mlapp + name: certificate spec: domains: - - mlbot.net \ No newline at end of file + - label-bot-prod.mlbot.net \ No newline at end of file diff --git a/deployment/overlays/prod/deployment.yaml b/deployment/overlays/prod/deployment.yaml index 0054c116..a9be9d84 100644 --- a/deployment/overlays/prod/deployment.yaml +++ b/deployment/overlays/prod/deployment.yaml @@ -4,6 +4,7 @@ metadata: name: ml-github-app spec: replicas: 9 + template: spec: containers: - name: frontend @@ -24,9 +25,10 @@ spec: value: "27079" # Pato the GitHub app PEM key - name: GITHUB_APP_PEM_KEY - value: /var/secrets/github/kf-label-bot-dev.private-key.pem + value: /var/secrets/github/kf-label-bot-prod.private-key.pem # The GCP project and pubsub topic to publish to should - # correspond to the production backend + # correspond to the production backend. The project is unfortunately + # named issue-label-bot-dev. - name: GCP_PROJECT_ID value: issue-label-bot-dev - name: GCP_PUBSUB_TOPIC_NAME diff --git a/deployment/overlays/prod/ingress.yaml b/deployment/overlays/prod/ingress.yaml new file mode 100644 index 00000000..54dc0b31 --- /dev/null +++ b/deployment/overlays/prod/ingress.yaml @@ -0,0 +1,7 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: frontend + annotations: + kubernetes.io/ingress.global-static-ip-name: label-bot-prod + networking.gke.io/managed-certificates: certificate diff --git a/deployment/overlays/prod/kustomization.yaml b/deployment/overlays/prod/kustomization.yaml new file mode 100644 index 00000000..d20645ee --- /dev/null +++ b/deployment/overlays/prod/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + environment: prod +namespace: label-bot-prod +images: +- name: gcr.io/github-probots/label-bot-frontend + newName: gcr.io/github-probots/label-bot-frontend + newTag: 2cb624e +resources: +- certificate.yaml +- ../../base +patchesStrategicMerge: +- deployment.yaml +- ingress.yaml diff --git a/deployment/service-test.yaml b/deployment/service-test.yaml deleted file mode 100644 index 11dc501d..00000000 --- a/deployment/service-test.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ml-github-app-test - namespace: mlapp - labels: - app: ml-github-app -spec: - ports: - - port: 3000 - protocol: TCP - selector: - app: ml-github-app - type: NodePort diff --git a/deployment/service.yaml b/deployment/service.yaml deleted file mode 100644 index f3cd8895..00000000 --- a/deployment/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ml-github-app - namespace: mlapp - labels: - app: ml-github-app -spec: - ports: - - port: 3000 - protocol: TCP - selector: - app: ml-github-app - type: NodePort \ No newline at end of file diff --git a/script/create_secrets.py b/script/create_secrets.py index ee5f17d5..44ab3564 100644 --- a/script/create_secrets.py +++ b/script/create_secrets.py @@ -15,8 +15,12 @@ import re import subprocess +DEV_ENVIRONMENT = "dev" +PROD_ENVIRONMENT = "prod" + # The namespace for the dev environment. DEV_NAMESPACE = "label-bot-dev" +PROD_NAMESPACE = "label-bot-prod" GCS_REGEX = re.compile("gs://([^/]*)(/.*)?") @@ -100,42 +104,60 @@ def copy_secret(source, dest): subprocess.check_call(command) @staticmethod - def create_dev(): - """Create the secrets for the dev environment.""" + def create(env): + """Create the secrets for the dev environment. + + Args: + env: The environment to create the secrets in. + """ + + if env == DEV_ENVIRONMENT: + namespace = DEV_NAMESPACE + github_app_pem_key = ("gs://issue-label-bot-dev_secrets/" + "kf-label-bot-dev.2019-12-30.private-key.pem") + webhook_gcs = ("gs://issue-label-bot-dev_secrets/" + "kf-label-bot-dev.webhook.secret") + elif env == PROD_ENVIRONMENT: + namespace = PROD_NAMESPACE + github_app_pem_key = ("gs://github-probots_secrets/" + "issue-label-bot-github-app.private-key.pem") + webhook_gcs = ("gs://github-probots_secrets/" + "issue-label-bot-prod.webhook.secret") + else: + raise ValueError(f"env={env} is not an allowed value; must be " + f"{DEV_ENVIRONMENT} or {PROD_ENVIRONMENT}") k8s_config.load_kube_config(persist_config=False) client = k8s_client.ApiClient() - if secret_exists(DEV_NAMESPACE, "user-gcp-sa", client): - logging.warning(f"Secret {DEV_NAMESPACE}/user-gcp-sa already exists; " + if secret_exists(namespace, "user-gcp-sa", client): + logging.warning(f"Secret {namespace}/user-gcp-sa already exists; " f"Not recreating it.") else: # We get a GCP secret by copying it from the kubeflow namespace. SecretCreator.copy_secret("kubeflow/user-gcp-sa", - f"{DEV_NAMESPACE}/user-gcp-sa") + f"{namespace}/user-gcp-sa") - if secret_exists(DEV_NAMESPACE, "github-app", client): - logging.warning(f"Secret {DEV_NAMESPACE}/github-app already exists; " + if secret_exists(namespace, "github-app", client): + logging.warning(f"Secret {namespace}/github-app already exists; " f"Not recreating it.") else: # Create the secret containing the PEM key for the github app - SecretCreator._secret_from_gcs(f"{DEV_NAMESPACE}/github-app", - "gs://issue-label-bot-dev_secrets/kf-label-bot-dev.2019-12-30.private-key.pem") + SecretCreator._secret_from_gcs(f"{namespace}/github-app", github_app_pem_key) # Create the inference secret containing the postgres database with # postgres secret and the webhook secret inference_secret = "ml-app-inference-secret" - if secret_exists(DEV_NAMESPACE, inference_secret, client): - logging.warning(f"Secret {DEV_NAMESPACE}/{inference_secret} already exists; " + if secret_exists(namespace, inference_secret, client): + logging.warning(f"Secret {namespace}/{inference_secret} already exists; " f"Not recreating it.") else: postgres = _read_gcs_path("gs://issue-label-bot-dev_secrets/" "issue-label-bot.postgres") - webhook = _read_gcs_path("gs://issue-label-bot-dev_secrets/" - "kf-label-bot-dev.webhook.secret") + webhook = _read_gcs_path(webhook_gcs) - subprocess.check_call(["kubectl", "-n", DEV_NAMESPACE, "create", + subprocess.check_call(["kubectl", "-n", namespace, "create", "secret", "generic", inference_secret, f"--from-literal=DATABASE_URL={postgres}", diff --git a/script/send_request.py b/script/send_request.py index 9f8328c5..90f6790d 100644 --- a/script/send_request.py +++ b/script/send_request.py @@ -17,16 +17,23 @@ def send(url="https://label-bot-dev.mlbot.net/event_handler"): secret_decoded = base64.b64decode(secret).decode() + if url != "https://label-bot-dev.mlbot.net/event_handler": + logging.error("You aren't using the dev label bot webhook but " + "send_request.py currently hard codes the GitHub App " + "Install ID to kf-label-bot-dev on kubeflow/code-intelligence " + "we need to change that in order to be able to allow " + "worker to actually write to the repo.") # TODO(jlewi): We should allow specificing a specific issue. payload = { "action": "opened", # Installation corresponding to kf-label-bot-dev on # kubeflow/code-intelligence "installation": { + # TODO(jlewi): This is the installation id of the kf-label-bot-dev "id": 5980888, }, "issue": { - "number": 99, + "number": 104, "title": "Test kf-label bot-dev this is a bug", "body": ("Test whether events are correctly routed to the dev instance." "If not then there is a bug in the setup") @@ -49,7 +56,10 @@ def send(url="https://label-bot-dev.mlbot.net/event_handler"): # We use data and not json because we need to compute the hash of the # data to match the signature - requests.post(url, data=data, headers=headers) + logging.info(f"Send url: {url}") + response = requests.post(url, data=data, headers=headers) + + logging.info(f"Response {response}") if __name__ == "__main__": logging.basicConfig(level=logging.INFO,