cleaning up MR, still need to fix and add tests

spotify · Aug 30, 2021 · d340099 · d340099
1 parent aa368d6
commit d340099
Show file tree

Hide file tree

Showing 4 changed files with 309 additions and 87 deletions.
diff --git a/examples/kubernetes.py b/examples/kubernetes.py
@@ -44,18 +44,31 @@
 # import luigi
 from luigi.contrib.kubernetes import KubernetesJobTask
 
-
 class PerlPi(KubernetesJobTask):
 
     name = "pi"                         # The name (prefix) of the job that will be created for identification purposes
-    backoff_limit = 2                   # This is the number of retries incase there is a pod/node/code failure
-    kubernetes_namespace = "my_nshere"  # This is the kubernetes namespace you wish to run in
-    print_pod_logs_on_exit = True       # Set this to true if you wish to see the logs of this
-    spec_schema = {                     # This is the standard "spec" of the containers in this job, this is a good sane example
+    kubernetes_namespace = "default"    # This is the kubernetes namespace you wish to run, if not specified it uses "default"
+    labels = {"job_name": "pi"}         # This is to add labels in Kubernetes to help identify it, this is on top of the internal luigi labels
+    backoff_limit = 2                   # This is the number of retries incase there is a pod/node/code failure, default 6
+    poll_interval = 1                   # To poll more regularly reduce this number, default 5
+    print_pod_logs_on_exit = False      # Set this to True if you wish to see the logs of this run after completion, False by default
+    print_pod_logs_during_run = True    # Set this to True if you wish to see the logs of this run while it is running, False by default
+    delete_on_success = True            # Set this to False to keep the job after it finishes successfully, for debugging purposes, True by default
+    spec_schema = {                     # This is the standard "spec" of the containers in this job, this is a good sane example with resource requests/limits
         "containers": [{
             "name": "pi",
             "image": "perl",
-            "command": ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
+            "command": ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"],
+            "resources": {
+                "requests": {
+                    "cpu": "50m",
+                    "memory": "50Mi"
+                },
+                "limits": {
+                    "cpu": "100m",
+                    "memory": "100Mi"
+                }
+            }
         }]
     }