MetadataStores error

[CongaJAMM]

Environment details

• OS type and version: Windows 11 OS build: 26100.7171
• Python version: 3.11.15
• pip version: pip 24.3.1
• google-cloud-aiplatform version: 1.161.0

Steps to reproduce

1. Had the same issue with the name attribute when using PipelineJob.from_pipeline_func() then I switched to the direct complilation method with the compiler.
2. Using uv with python version 3.11.15; Dependencies kfp=2.17.0, google-cloud-pipeline-components=2.22.0, google-cloud-aiplatform=1.161.0
3. Executed script with "uv run python -m main"
4. Made sure execution_service account had the following roles: ai platform admin and service account user.

Code example

#main.py
"""Automated Serverless trigger for a Gemini Enterprise Agent Platform Pipeline."""
import kfp
# import functions_framework
from google.cloud import aiplatform
from google_cloud_pipeline_components.v1.custom_job import CustomTrainingJobOp
from kfp import compiler

gc_project_id = "your-gcp-project-id"
gcs_bucket = "your-gcs-bucket-name"
gcs_project_dir = "YOUR-PROJECT-NAME"
pipeline_root_path = f"gs://{gcs_bucket}/{gcs_project_dir}"
workspace_dir = "Workspace-Dir"
staging_dir = "Staging"
gcp_staging_bucket = f"{pipeline_root_path}/{staging_dir}"
gcp_output_dir = f"{pipeline_root_path}/{workspace_dir}"
location = "YOUR-LOCATION"
execution_sa = "your-project-number-compute@developer.gserviceaccount.com"

image_registry = "YOUR-LOCATION-docker.pkg.dev"
image_repository = "your-repository-name"
image_prefix = f"{image_registry}/{gc_project_id}/{image_repository}"

preprocess_image_name = f"{image_prefix}/preprocess:latest"
training_image_name = f"{image_prefix}/training:latest"

@kfp.dsl.pipeline(
    name="vertex-ai-pipeline",
    description="First Google Cloud Enterprise Agent Pipeline",
    pipeline_root=None, # "./"
    display_name="First-Pipeline",
    pipeline_config=None
)
def vertex_pipeline(
    message: str
):
    print("Pipeline Message: %s", message)

    # from custom_training_job in `components/google-cloud/google_cloud_pipeline_components/v1/custom_job/component.py`
    preprocessing_component = CustomTrainingJobOp(
        display_name="Preprocessing-Component-Job",
        # location=None,
        worker_pool_specs=[{
            "machine_spec": {"machine_type": "n1-standard-4"},
            "replica_count": 1,
            "container_spec": {"image_uri": preprocess_image_name}
        }],
    )
def trigger_ml_pipeline() -> None:
    # Authenticates and sets critical settings with Centralized Google Cloud Global configuration object
        # `google/cloud/aiplatform/initializer.py` > _Config.init(...)
        # made singleton from `google/cloud/aiplatform/__init__.py`
    aiplatform.init(
        project=gc_project_id,
        location=location,
        experiment=None,
        experiment_description=None,
        experiment_tensorboard=None,
        staging_bucket=gcp_staging_bucket,
        credentials=None,
        encryption_spec_key_name=None,
        network=None,
        service_account=execution_sa,   # SET SERVICE ACCOUNT HERE
        api_endpoint=None,
        api_transport=None,
        request_metadata=None
        )

    compiler.Compiler().compile(
        pipeline_func=vertex_pipeline,
        package_path="./single-run.json",
        pipeline_name="vertex-ai-pipeline",
        pipeline_display_name="first-pipeline-run",
        pipeline_parameters={
            # Adjust to custom pipeline function variables
            "message": "Hello-world from console function call!!",
        },
        type_check=True,
        # kubernetes_manifest_format=None,
        # kubernetes_manifest_options=None
    )

    pipeline_job = aiplatform.PipelineJob(
        display_name="First Pipeline Job",
        template_path="./single-run.json",
        job_id=None,
        pipeline_root=pipeline_root_path,
        parameter_values=None,
        input_artifacts=None,
        enable_caching=False,
        encryption_spec_key_name=None,
        labels=None,
        credentials=None,
        project=None,
        location=None,
        failure_policy=None,
    )

    pipeline_job.run(
        service_account=execution_sa,
        network= None,
        reserved_ip_ranges= None,
        create_request_timeout= None,
        enable_preflight_validations=False,
    )

Stack trace

C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\Google-Cloud-Functions\trigger-platform-pipeline>uv run python -m main
C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\.venv\Lib\site-packages\google\cloud\aiplatform\models.py:52: FutureWarning: Support for google-cloud-storage < 3.0.0 will be removed in a future version of google-cloud-aiplatform. Please upgrade to google-cloud-storage >= 3.0.0.
  from google.cloud.aiplatform.utils import gcs_utils
Pipeline Message: %s {{channel:task=;name=message;type=String;}}
Creating PipelineJob
PipelineJob created. Resource name: projects/613982306205/locations/us-west2/pipelineJobs/vertex-ai-pipeline-20260724122251
To use this PipelineJob in another session:
pipeline_job = aiplatform.PipelineJob.get('projects/613982306205/locations/us-west2/pipelineJobs/vertex-ai-pipeline-20260724122251')
View Pipeline Job:
https://console.cloud.google.com/vertex-ai/locations/us-west2/pipelines/runs/vertex-ai-pipeline-20260724122251?project=613982306205
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\Google-Cloud-Functions\trigger-platform-pipeline\ye.py", line 157, in <module>
    trigger_ml_pipeline()
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\Google-Cloud-Functions\trigger-platform-pipeline\ye.py", line 140, in trigger_ml_pipeline
    pipeline_job.run(
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\.venv\Lib\site-packages\google\cloud\aiplatform\pipeline_jobs.py", line 334, in run
    self._run(
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\.venv\Lib\site-packages\google\cloud\aiplatform\base.py", line 862, in wrapper
    return method(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\.venv\Lib\site-packages\google\cloud\aiplatform\pipeline_jobs.py", line 382, in _run
    self._block_until_complete()
  File "C:\Users\brianperez\Desktop\DEV\Machine-Learning-Projects\Ames-Housing\.venv\Lib\site-packages\google\cloud\aiplatform\pipeline_jobs.py", line 799, in _block_until_complete
    raise RuntimeError("Job failed with:\n%s" % self._gca_resource.error)
RuntimeError: Job failed with:
code: 7
message: "Failed to create pipeline job. Error: Permission \'aiplatform.metadataStores.get\' denied on resource \'//aiplatform.googleapis.com/projects/613982306205/locations/us-west2/metadataStores/default\' (or it may not exist). Remediate access with this Troubleshooter URL or share it with your administrator - https://console.cloud.google.com/iam-admin/troubleshooter/summary;errorId=CiQwMTlmOTU5NC1hMGFhLTc4YjUtOGRjZS00ZjM1OTYzYmMxZDASP3Byb2plY3RzLzYxMzk4MjMwNjIwNS9sb2NhdGlvbnMvdXMtd2VzdDIvbWV0YWRhdGFTdG9yZXMvZGVmYXVsdA%3D%3D .."

According to Google Cloud Docs, the metadata store should create itself on the first run.

I found this stack from StackOverflow but it did not resolve my issue.