Merge pull request #33 from GoogleCloudPlatform/feature/model_monitoring

Feature/model monitoring

AutoMLOps/AutoMLOps.py

@@ -246,7 +246,6 @@ def _push_to_csr():
         raise RuntimeError(f'Expected remote origin url {csr_remote_origin_url} but found {actual_remote}. Reset your remote origin url to continue.')
 
     # Add, commit, and push changes to CSR
-    execute_process(f'touch {BASE_DIR}scripts/pipeline_spec/.gitkeep', to_null=False) # needed to keep dir here
     execute_process('git add .', to_null=False)
     execute_process('''git commit -m 'Run AutoMLOps' ''', to_null=False)
     execute_process(f'''git push origin {defaults['gcp']['cloud_source_repository_branch']} --force''', to_null=False)

AutoMLOps/deployments/cloudbuild/constructs/scripts.py

@@ -91,16 +91,16 @@ def _create_kfp_cloudbuild_config(self):
             f'''    args: [ "build", "-t", "{self.__af_registry_location}-docker.pkg.dev/{self.__project_id}/{self.__af_registry_name}/components/component_base:latest", "." ]\n'''
             f'''    dir: "{self.__base_dir}components/component_base"\n'''
             f'''    id: "build_component_base"\n'''
-            f'''    waitFor: ["-"]\n'''
+            f'''    waitFor: ["-"]\n''')
+
+        cloudbuild_cloudrun_config = (
             f'\n'
             f'''  # build the run_pipeline image\n'''
             f'''  - name: 'gcr.io/cloud-builders/docker'\n'''
             f'''    args: [ "build", "-t", "{self.__af_registry_location}-docker.pkg.dev/{self.__project_id}/{self.__af_registry_name}/run_pipeline:latest", "-f", "cloud_run/run_pipeline/Dockerfile", "." ]\n'''
             f'''    dir: "{self.__base_dir}"\n'''
             f'''    id: "build_pipeline_runner_svc"\n'''
-            f'''    waitFor: ['build_component_base']\n''')
-
-        cloudbuild_cloudrun_config = (
+            f'''    waitFor: ['build_component_base']\n'''
             f'\n'
             f'# ==============================================================================\n'
             f'# PUSH & DEPLOY CUSTOM IMAGES\n'

AutoMLOps/frameworks/kfp/builder.py

@@ -126,6 +126,12 @@ def build(project_id: str,
         build_component(path)
     build_pipeline(custom_training_job_specs, pipeline_params)
 
+    # Write empty .gitkeep to pipeline_spec directory
+    write_file(f'{BASE_DIR}scripts/pipeline_spec/.gitkeep', '', 'w')
+
+    # Write empty .gitkeep to pipeline_spec directory
+    write_file(f'{BASE_DIR}README.md', kfp_scripts.readme, 'w')
+
     # Write dockerfile to the component base directory
     write_file(f'{GENERATED_COMPONENT_BASE}/Dockerfile', kfp_scripts.dockerfile, 'w')
 

AutoMLOps/frameworks/kfp/constructs/scripts.py

@@ -119,6 +119,7 @@ def __init__(self,
         self.dockerfile = self._create_dockerfile()
         self.defaults = self._create_default_config()
         self.requirements = self._create_requirements()
+        self.readme = self._create_generated_readme()
 
     def _build_pipeline_spec(self):
         """Builds content of a shell script to build the pipeline specs.
@@ -399,7 +400,7 @@ def _create_dockerfile(self):
             f'ENTRYPOINT ["/bin/bash"]\n')
 
     def _create_default_config(self):
-        """Creates default defaults.yaml file contents. This defaults
+        """Creates defaults.yaml file contents. This defaults
         file is used by subsequent functions and by the pipeline
         files themselves.
 
@@ -502,3 +503,55 @@ def _create_requirements(self):
         # Stringify and sort
         reqs_str = ''.join(r+'\n' for r in sorted(set_of_requirements))
         return reqs_str
+
+    def _create_generated_readme(self):
+        """Creates a readme markdown file to describe the contents of the
+        generated AutoMLOps code repo.
+
+        Returns:
+            str: readme.md file content
+        """
+        cloud_run_dirs = ''
+        if not self._run_local:
+            cloud_run_dirs = (
+                '├── cloud_run                                      : Cloud Runner service for submitting PipelineJobs.\n'
+                '    ├──run_pipeline                                : Contains main.py file, Dockerfile and requirements.txt\n'
+                '    ├──queueing_svc                                : Contains files for scheduling and queueing jobs to runner service\n'
+            )
+
+        return (
+            '# AutoMLOps - Generated Code Directory\n'
+            '\n'
+            '**Note: This directory contains code generated using AutoMLOps**\n'
+            '\n'
+            'AutoMLOps is a service that generates a production ready MLOps pipeline from Jupyter Notebooks, bridging the gap between Data Science and DevOps and accelerating the adoption and use of Vertex AI. The service generates an MLOps codebase for users to customize, and provides a way to build and manage a CI/CD integrated MLOps pipeline from the notebook. AutoMLOps automatically builds a source repo for versioning, cloudbuild configs and triggers, an artifact registry for storing custom components, gs buckets, service accounts and updated IAM privs for running pipelines, enables APIs (cloud Run, Cloud Build, Artifact Registry, etc.), creates a runner service API in Cloud Run for submitting PipelineJobs to Vertex AI, and a Cloud Scheduler job for submitting PipelineJobs on a recurring basis. These automatic integrations empower data scientists to take their experiments to production more quickly, allowing them to focus on what they do best: providing actionable insights through data.\n'
+            '\n'
+            '# User Guide\n'
+            '\n'
+            'For a user-guide, please view these [slides](https://github.com/GoogleCloudPlatform/automlops/blob/main/AutoMLOps_Implementation_Guide_External.pdf).\n'
+            '\n'
+            '# Layout\n'
+            '\n'
+            '```bash\n'
+            '.\n'
+            f'{cloud_run_dirs}'
+            '├── components                                     : Custom vertex pipeline components.\n'
+            '    ├──component_base                              : Contains all the python files, Dockerfile and requirements.txt\n'
+            '    ├──component_a                                 : Components generated using AutoMLOps\n'
+            '    ├──...\n'
+            '├── images                                         : Custom container images for training models.\n'
+            '├── pipelines                                      : Vertex ai pipeline definitions.\n'
+            '    ├── pipeline.py                                : Full pipeline definition.\n'
+            '    ├── pipeline_runner.py                         : Sends a PipelineJob to Vertex AI.\n'
+            '    ├── runtime_parameters                         : Variables to be used in a PipelineJob.\n'
+            '        ├── pipeline_parameter_values.json         : Json containing pipeline parameters.\n'  
+            '├── configs                                        : Configurations for defining vertex ai pipeline.\n'
+            '    ├── defaults.yaml                              : PipelineJob configuration variables.\n'
+            '├── scripts                                        : Scripts for manually triggering the cloud run service.\n'
+            '    ├── build_components.sh                        : Submits a Cloud Build job that builds and deploys the components.\n'
+            '    ├── build_pipeline_spec.sh                     : Builds the pipeline specs.\n'
+            '    ├── create_resources.sh                        : Creates an artifact registry and gs bucket if they do not already exist.\n'
+            '    ├── run_pipeline.sh                            : Submit the PipelineJob to Vertex AI.\n'
+            '    ├── run_all.sh                                 : Builds components, pipeline specs, and submits the PipelineJob.\n'
+            '└── cloudbuild.yaml                                : Cloudbuild configuration file for building custom components.\n'
+            '```\n')

AutoMLOps/utils/utils.py

@@ -243,8 +243,8 @@ def update_params(params: list) -> list:
         str: 'String',
         float: 'Float',
         bool: 'Bool',
-        list: 'List',
-        dict: 'Dict'
+        list: 'JsonArray',
+        dict: 'JsonObject'
     }
     for param in params:
         try:

CHANGELOG.md

@@ -4,10 +4,15 @@ All notable changes to this project will be documented in this file.
 ## [1.1.4] - 2023-07-25
 
 ### Added
+
+- Writes .gitkeep to scripts/pipeline_spec directory by default
+- Generates a readme.md into generated AutoMLOps codebase now by default
 
 ### Changed
 
 - Two newlines after functions (linting)
+- Parameter mapping (list -> JsonArray, map -> JsonObject)
+- Updated documentation: added examples section into main readme, changed package version deps in examples notebooks
 
 ### Fixed
 

README.md

@@ -82,6 +82,20 @@ AutoMLOps will update [IAM privileges](https://cloud.google.com/iam/docs/underst
 
 For a user-guide, please view these [slides](./AutoMLOps_Implementation_Guide_External.pdf).
 
+# List of Examples
+
+Training
+- [00_introduction_training_example](./examples/training/00_introduction_training_example.ipynb)
+- [00_introduction_training_example_no_notebook](./examples/training/00_introduction_training_example_no_notebook.py)
+- [01_clustering_example](./examples/training/01_clustering_example.ipynb)
+- [02_tensorflow_transfer_learning_gpu_example](./examples/training/02_tensorflow_transfer_learning_gpu_example.ipynb)
+- [03_bqml_introduction_training_example](./examples/training/03_bqml_introduction_training_example.ipynb)
+- [04_bqml_forecasting-retail-demand](./examples/training/04_bqml_forecasting-retail-demand.ipynb)
+
+Inferencing
+- [00_batch_prediction_example](./examples/inferencing/00_batch_prediction_example.ipynb)
+- [01_customer_churn_model_monitoring_example](./examples/inferencing/01_customer_churn_model_monitoring_example.ipynb)
+
 # Options
 
 AutoMLOps CI/CD options:
@@ -216,10 +230,8 @@ The [example notebook](./examples/training/00_training_example.ipynb) comes with
 - [Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
 
 # Next Steps / Backlog
-- Refine unit tests
 - Use [terraform](https://github.com/GoogleCloudPlatform/vertex-pipelines-end-to-end-samples/tree/main/terraform) for the creation of resources.
 - Allow multiple AutoMLOps pipelines within the same directory
-- Adding model monitoring part
 - Alternatives to Pipreqs
 
 # Contributors

examples/inferencing/00_batch_prediction_example.ipynb

@@ -23,7 +23,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "8790d7ed",
    "metadata": {},
@@ -32,18 +31,18 @@
     "\n",
     "<table align=\"left\">\n",
     "  <td>\n",
-    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/automlops/blob/main/example/automlops_example_notebook.ipynb\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/automlops/blob/main/examples/inference/00_batch_prediction_example.ipynb\">\n",
     "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
     "    </a>\n",
     "  </td>\n",
     "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/automlops/blob/main/example/automlops_example_notebook.ipynb\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/automlops/blob/main/examples/inference/00_batch_prediction_example.ipynb\">\n",
     "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
     "      View on GitHub\n",
     "    </a>\n",
     "  </td>\n",
     "  <td>\n",
-    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/automlops/main/example/automlops_example_notebook.ipynb\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/automlops/main/examples/inference/00_batch_prediction_example.ipynb\">\n",
     "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
     "      Open in Vertex AI Workbench\n",
     "    </a>\n",
@@ -53,7 +52,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "6f938540",
    "metadata": {},
@@ -66,7 +64,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "22881a6a",
    "metadata": {},
@@ -99,7 +96,7 @@
     "- `docopt==0.6.2`,\n",
     "- `docstring-parser==0.15`,\n",
     "- `pipreqs==0.4.11`,\n",
-    "- `PyYAML==5.4.1`,\n",
+    "- `PyYAML==6.0.1`,\n",
     "- `yarg==0.1.9`\n",
     "\n",
     "# APIs & IAM\n",
@@ -158,7 +155,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "12381413",
    "metadata": {},
@@ -168,7 +164,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "a231b629",
    "metadata": {},
@@ -189,7 +184,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "d5e4d190",
    "metadata": {},
@@ -210,7 +204,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "17db55d5",
    "metadata": {},
@@ -239,7 +232,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "250d511b",
    "metadata": {},
@@ -299,7 +291,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "503f4a85-c357-4ec4-8ea5-0eeb2c8afd39",
    "metadata": {},
@@ -328,17 +319,15 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "7b3fe994",
    "metadata": {},
    "source": [
     "# 1. AutoMLOps Pipeline\n",
-    "This workflow will define and generate a pipeline without using Kubeflow spec. `generate()` will create all the necessary files but not run them. `go()` will create all the necessary files, resources, push the code to the source repo to trigger the build, and then submit a Pipeline training job to Vertex AI. Please see the [readme](https://github.com/GoogleCloudPlatform/automlops/blob/main/README.md) for more information."
+    "This workflow will define and generate a pipeline using AutoMLOps. `generate()` will create all the necessary files but not run them. `go()` will create all the necessary files, resources, push the code to the source repo to trigger the build, and then submit a Pipeline training job to Vertex AI. Please see the [readme](https://github.com/GoogleCloudPlatform/automlops/blob/main/README.md) for more information."
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "2219ee67",
    "metadata": {},
@@ -349,15 +338,41 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "cd5ac7db",
+   "id": "3d02ea3e",
    "metadata": {},
    "outputs": [],
    "source": [
     "from AutoMLOps import AutoMLOps"
    ]
   },
   {
-   "attachments": {},
+   "cell_type": "markdown",
+   "id": "cbc81099",
+   "metadata": {},
+   "source": [
+    "## Clear the cache\n",
+    "`AutoMLOps.clear_cache` will remove previous instantiations of AutoMLOps components and pipelines. Use this function if you have previously defined a component that you no longer need."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "85d5e260",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cache cleared.\n"
+     ]
+    }
+   ],
+   "source": [
+    "AutoMLOps.clear_cache()"
+   ]
+  },
+  {
    "cell_type": "markdown",
    "id": "d4476cb4-91c5-42ff-a500-8cc275fedbd1",
    "metadata": {},
@@ -368,7 +383,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "id": "5371be73-db3f-4d79-bde7-94fcd5ea13b2",
    "metadata": {},
    "outputs": [],
@@ -469,7 +484,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "73d96dcb-a020-4bab-b0e3-1e32f6b2aecf",
    "metadata": {},
@@ -480,7 +494,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "id": "a36e0154",
    "metadata": {},
    "outputs": [],
@@ -518,7 +532,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "3874f4f6",
    "metadata": {},
@@ -528,7 +541,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "id": "4cb3786b",
    "metadata": {},
    "outputs": [],
@@ -553,7 +566,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "c1e0aa05",
    "metadata": {},
@@ -564,7 +576,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "id": "db51a8d0",
    "metadata": {},
    "outputs": [