removed ml.googleapis.com from APIs list

CHANGELOG.md

@@ -1,6 +1,20 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [1.2.2] - 2023-09-25
+
+### Added
+
+### Changed
+
+- Commented out change from 1.2.1 to set cloud functions default registry to artifact-registry; there appears to be an auth bug with this setting
+- Updated documentation
+
+### Fixed
+
+- Removed legacy ml.googleapis.com API that is deprecated as of 9/25/2023
+
+
 ## [1.2.1] - 2023-09-20
 
 ### Added

README.md

@@ -169,7 +169,6 @@ Based on the above user selection, AutoMLOps will enable up to the following API
 - [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)
 - [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)
 - [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)
-- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)
 - [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)
 - [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)
 - [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)

examples/inferencing/00_batch_prediction_example.ipynb

@@ -104,7 +104,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -113,7 +112,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/inferencing/01_customer_churn_model_monitoring_example.ipynb

@@ -134,7 +134,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -143,7 +142,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb

@@ -106,7 +106,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -115,7 +114,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/training/.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+#!include:.gitignore

examples/training/00_introduction_training_example.ipynb

@@ -103,7 +103,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -112,7 +111,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",
@@ -684,56 +689,58 @@
    "cell_type": "code",
    "execution_count": 13,
    "id": "a7e1efd9",
-   "metadata": {},
+   "metadata": {
+    "scrolled": false
+   },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "WARNING: Provisioning requires these permissions:\n",
+      "-iam.serviceAccounts.list\n",
+      "-storage.buckets.create\n",
       "-cloudscheduler.jobs.create\n",
-      "-iam.serviceAccounts.create\n",
       "-serviceusage.services.use\n",
-      "-serviceusage.services.enable\n",
-      "-artifactregistry.repositories.create\n",
-      "-source.repos.list\n",
-      "-artifactregistry.repositories.list\n",
-      "-cloudbuild.builds.list\n",
-      "-iam.serviceAccounts.list\n",
-      "-pubsub.topics.list\n",
+      "-pubsub.subscriptions.create\n",
       "-cloudbuild.builds.create\n",
-      "-storage.buckets.create\n",
       "-pubsub.topics.create\n",
       "-cloudfunctions.functions.create\n",
-      "-cloudfunctions.functions.get\n",
-      "-pubsub.subscriptions.create\n",
-      "-iam.serviceAccounts.actAs\n",
-      "-storage.buckets.get\n",
       "-resourcemanager.projects.setIamPolicy\n",
-      "-cloudscheduler.jobs.list\n",
+      "-storage.buckets.get\n",
+      "-iam.serviceAccounts.actAs\n",
+      "-iam.serviceAccounts.create\n",
       "-source.repos.create\n",
       "-pubsub.subscriptions.list\n",
+      "-cloudscheduler.jobs.list\n",
+      "-artifactregistry.repositories.create\n",
+      "-serviceusage.services.enable\n",
+      "-cloudfunctions.functions.get\n",
+      "-source.repos.list\n",
+      "-pubsub.topics.list\n",
+      "-cloudbuild.builds.list\n",
+      "-artifactregistry.repositories.list\n",
       "\n",
       "You are currently using: [email protected]. Please check your account permissions.\n",
       "The following are the recommended roles for provisioning:\n",
-      "-roles/artifactregistry.admin\n",
+      "-roles/iam.serviceAccountUser\n",
       "-roles/serviceusage.serviceUsageAdmin\n",
-      "-roles/cloudbuild.builds.editor\n",
+      "-roles/cloudfunctions.admin\n",
+      "-roles/storage.admin\n",
       "-roles/source.admin\n",
-      "-roles/iam.serviceAccountUser\n",
-      "-roles/cloudscheduler.admin\n",
       "-roles/iam.serviceAccountAdmin\n",
-      "-roles/storage.admin\n",
+      "-roles/cloudscheduler.admin\n",
+      "-roles/artifactregistry.admin\n",
       "-roles/pubsub.editor\n",
+      "-roles/cloudbuild.builds.editor\n",
       "-roles/resourcemanager.projectIamAdmin\n",
-      "-roles/cloudfunctions.admin\n",
       "\n",
       "\u001b[0;32m Setting up API services in project automlops-sandbox \u001b[0m\n",
-      "Operation \"operations/acat.p2-45373616427-2910359e-ca81-4af7-997d-7452e76b8677\" finished successfully.\n",
+      "Operation \"operations/acat.p2-45373616427-a14841f0-e60d-4c6c-ace7-073e858c1888\" finished successfully.\n",
       "\u001b[0;32m Setting up Artifact Registry in project automlops-sandbox \u001b[0m\n",
       "Listing items under project automlops-sandbox, location us-central1.\n",
       "\n",
-      "dry-beans-dt-artifact-registry    DOCKER  STANDARD_REPOSITORY  Artifact Registry dry-beans-dt-artifact-registry in us-central1.                            us-central1                                  Google-managed key  2023-09-05T11:25:48  2023-09-20T13:10:52  9080.502\n",
+      "dry-beans-dt-artifact-registry    DOCKER  STANDARD_REPOSITORY  Artifact Registry dry-beans-dt-artifact-registry in us-central1.                            us-central1                                  Google-managed key  2023-09-05T11:25:48  2023-09-20T14:07:57  9794.524\n",
       "Artifact Registry: dry-beans-dt-artifact-registry already exists in project automlops-sandbox\n",
       "\u001b[0;32m Setting up Storage Bucket in project automlops-sandbox \u001b[0m\n",
       "gs://automlops-sandbox-dry-beans-dt-bucket/\n",
@@ -751,11 +758,11 @@
       "\u001b[0;32m Deploying Cloud Functions: dry-beans-dt-job-submission-svc in project automlops-sandbox \u001b[0m\n",
       "Deploying function (may take a while - up to 2 minutes)...\n",
       "..\n",
-      "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/02d7faae-e8b0-409b-84e2-cb2ad5c86164?project=45373616427\n",
-      "...........................................................done.\n",
+      "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/71111d1b-0448-4f22-862b-d040e37cedf2?project=45373616427\n",
+      "................................................................done.\n",
       "availableMemoryMb: 512\n",
-      "buildId: 02d7faae-e8b0-409b-84e2-cb2ad5c86164\n",
-      "buildName: projects/45373616427/locations/us-central1/builds/02d7faae-e8b0-409b-84e2-cb2ad5c86164\n",
+      "buildId: 71111d1b-0448-4f22-862b-d040e37cedf2\n",
+      "buildName: projects/45373616427/locations/us-central1/builds/71111d1b-0448-4f22-862b-d040e37cedf2\n",
       "dockerRegistry: ARTIFACT_REGISTRY\n",
       "entryPoint: process_request\n",
       "eventTrigger:\n",
@@ -770,11 +777,11 @@
       "name: projects/automlops-sandbox/locations/us-central1/functions/dry-beans-dt-job-submission-svc\n",
       "runtime: python39\n",
       "serviceAccountEmail: [email protected]\n",
-      "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/fce36a97-cf63-49c5-94ab-144dcf125cdd.zip\n",
+      "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/de8b497b-dff0-4ba2-b88c-d9b59f868941.zip\n",
       "status: ACTIVE\n",
       "timeout: 540s\n",
-      "updateTime: '2023-09-20T18:01:23.259Z'\n",
-      "versionId: '3'\n",
+      "updateTime: '2023-09-25T16:24:11.187Z'\n",
+      "versionId: '5'\n",
       "\u001b[0;32m Setting up Cloud Build Trigger in project automlops-sandbox \u001b[0m\n",
       "name: dry-beans-dt-build-trigger\n",
       "Cloudbuild Trigger already exists in project automlops-sandbox for repo dry-beans-dt-repository\n",

examples/training/01_clustering_example.ipynb

@@ -102,7 +102,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -111,7 +110,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/training/02_tensorflow_transfer_learning_gpu_example.ipynb

@@ -103,7 +103,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -112,7 +111,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/training/03_bqml_introduction_training_example.ipynb

@@ -104,7 +104,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -113,7 +112,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

examples/training/04_bqml_forecasting-retail-demand.ipynb

@@ -123,7 +123,6 @@
     "- [compute.googleapis.com](https://cloud.google.com/compute/docs/reference/rest/v1)\n",
     "- [iam.googleapis.com](https://cloud.google.com/iam/docs/reference/rest)\n",
     "- [iamcredentials.googleapis.com](https://cloud.google.com/iam/docs/reference/credentials/rest)\n",
-    "- [ml.googleapis.com](https://cloud.google.com/ai-platform/training/docs/reference/rest)\n",
     "- [pubsub.googleapis.com](https://cloud.google.com/pubsub/docs/reference/rest)\n",
     "- [run.googleapis.com](https://cloud.google.com/run/docs/reference/rest)\n",
     "- [storage.googleapis.com](https://cloud.google.com/storage/docs/apis)\n",
@@ -132,7 +131,13 @@
     "\n",
     "AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step:\n",
     "1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added:\n",
-    "- roles/aiplatform.serviceAgent\n",
+    "- roles/aiplatform.user\n",
+    "- roles/artifactregistry.reader\n",
+    "- roles/bigquery.user\n",
+    "- roles/bigquery.dataEditor\n",
+    "- roles/iam.serviceAccountUser\n",
+    "- roles/storage.admin\n",
+    "- roles/cloudfunctions.admin\n",
     "\n",
     "# User Guide\n",
     "\n",

google_cloud_automlops/__init__.py

@@ -23,6 +23,6 @@
 series of directories to support the creation of Vertex Pipelines.
 """
 # pylint: disable=invalid-name
-__version__ = '1.2.1'
+__version__ = '1.2.2'
 __author__ = 'Sean Rastatter'
 __credits__ = 'Google'

google_cloud_automlops/provisioning/gcloud/templates/provision_resources.sh.j2

@@ -143,7 +143,6 @@ fi
 echo -e "$GREEN Deploying Cloud Functions: ${PIPELINE_JOB_SUBMISSION_SERVICE_NAME} in project $PROJECT_ID $NC"
 gcloud functions deploy $PIPELINE_JOB_SUBMISSION_SERVICE_NAME \
   --no-allow-unauthenticated \
-  --docker-registry="artifact-registry" \
   --trigger-topic=$PUBSUB_TOPIC_NAME \
   --entry-point=process_request \
   --runtime=python39 \

google_cloud_automlops/provisioning/terraform/templates/environment/main.tf.j2

@@ -110,7 +110,7 @@ resource "google_storage_bucket_object" "archive_submission_service_code" {
 resource "google_cloudfunctions_function" "pipeline_job_submission_service" {
   name                  = var.pipeline_job_submission_service_name
   region                = var.pipeline_job_submission_service_location
-  docker_registry       = "ARTIFACT_REGISTRY"
+# docker_registry       = "ARTIFACT_REGISTRY"
   service_account_email = google_service_account.pipeline_job_runner_service_account.email
   runtime               = "python39"
   entry_point           = "process_request"

google_cloud_automlops/utils/utils.py

@@ -466,7 +466,6 @@ def get_required_apis(defaults: dict) -> Set:
         'compute.googleapis.com',
         'iamcredentials.googleapis.com',
         'iam.googleapis.com',
-        'ml.googleapis.com',
         'pubsub.googleapis.com',
         'storage.googleapis.com'])
     if defaults['tooling']['orchestration_framework'] == Orchestrator.KFP.value:

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name='google-cloud-automlops',
-    version='1.2.1',
+    version='1.2.2',
     description='Build MLOps Pipelines in Minutes.',
     long_description=readme_contents,
     long_description_content_type='text/markdown',