diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index d91d352dc1..ad9806a1f3 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -90,6 +90,21 @@ AGENT_DATA = _evals_constant.AGENT_DATA +def _local_timestamp() -> str: + """Returns the current local time as 'M/D/YYYY, H:MM:SS AM/PM'. + + Matches the Agent Platform UI's default experiment name timestamp format + (e.g. '6/1/2026, 1:12:29 PM'). + """ + now = datetime.datetime.now() + hour_12 = now.hour % 12 or 12 + meridiem = "AM" if now.hour < 12 else "PM" + return ( + f"{now.month}/{now.day}/{now.year}, " + f"{hour_12}:{now.minute:02d}:{now.second:02d} {meridiem}" + ) + + @contextlib.contextmanager def _temp_logger_level(logger_name: str, level: int) -> None: # type: ignore[misc] """Temporarily sets the level of a logger.""" diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 55d811edde..1444647de9 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -160,6 +160,13 @@ def _CreateEvaluationRunParameters_to_vertex( [item for item in getv(from_object, ["analysis_configs"])], ) + if getv(from_object, ["evaluation_experiment"]) is not None: + setv( + to_object, + ["evaluationExperiment"], + getv(from_object, ["evaluation_experiment"]), + ) + return to_object @@ -224,6 +231,20 @@ def _CustomCodeExecutionSpec_to_vertex( return to_object +def _DeleteEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _DeleteEvaluationMetricParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1091,6 +1112,48 @@ def _UnifiedMetric_to_vertex( return to_object +def _UpdateEvaluationExperimentConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["update_mask"]) is not None: + setv( + parent_object, ["_query", "updateMask"], getv(from_object, ["update_mask"]) + ) + + if getv(from_object, ["display_name"]) is not None: + setv(parent_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(parent_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(parent_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(parent_object, ["metadata"], getv(from_object, ["metadata"])) + + return to_object + + +def _UpdateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + _UpdateEvaluationExperimentConfig_to_vertex( + getv(from_object, ["config"]), to_object + ) + + return to_object + + class Evals(_api_module.BaseModule): def create_evaluation_experiment( @@ -1359,6 +1422,7 @@ def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -1373,6 +1437,7 @@ def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -1509,6 +1574,89 @@ def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _delete_evaluation_metric( self, *, @@ -2459,6 +2607,90 @@ def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("patch", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def evaluate_instances( self, *, @@ -2993,6 +3225,7 @@ def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfoOrDict] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -3013,6 +3246,11 @@ def create_evaluation_run( metrics: The list of metrics to evaluate. name: The name of the evaluation run. display_name: The display name of the evaluation run. + evaluation_experiment: The resource name of an existing + EvaluationExperiment to group this run under. If omitted, a new + EvaluationExperiment is created automatically so the run is visible in + the Agent Platform UI. Pass an existing experiment name to group + multiple runs together. agent_info: The agent info to evaluate. Mutually exclusive with `inference_configs`. agent: The agent resource name in str type. Accepts either an Agent @@ -3152,9 +3390,19 @@ def create_evaluation_run( ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name return self._create_evaluation_run( name=resolved_name, display_name=display_name or resolved_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, @@ -3796,6 +4044,7 @@ async def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -3810,6 +4059,7 @@ async def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -3950,6 +4200,91 @@ async def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + async def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _delete_evaluation_metric( self, *, @@ -4924,6 +5259,92 @@ async def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + async def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "patch", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def batch_evaluate( self, *, @@ -5053,6 +5474,40 @@ async def get_evaluation_run( return result + @_common.experimental_warning( + "The Vertex SDK GenAI evals.create_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + async def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return await self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + async def create_evaluation_run( self, *, @@ -5061,6 +5516,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -5081,6 +5537,11 @@ async def create_evaluation_run( metrics: The list of metrics to evaluate. name: The name of the evaluation run. display_name: The display name of the evaluation run. + evaluation_experiment: The resource name of an existing + EvaluationExperiment to group this run under. If omitted, a new + EvaluationExperiment is created automatically so the run is visible in + the Agent Platform UI. Pass an existing experiment name to group + multiple runs together. agent_info: The agent info to evaluate. Mutually exclusive with `inference_configs`. agent: The agent resource name in str type. Accepts either an Agent @@ -5220,10 +5681,20 @@ async def create_evaluation_run( ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name result = await self._create_evaluation_run( name=resolved_name, display_name=display_name or resolved_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index e7612f28d1..6027cc3bc0 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -56,6 +56,7 @@ from .common import _DeleteAgentEngineSessionRequestParameters from .common import _DeleteAgentEngineTaskRequestParameters from .common import _DeleteDatasetRequestParameters +from .common import _DeleteEvaluationExperimentParameters from .common import _DeleteEvaluationMetricParameters from .common import _DeleteMultimodalDatasetRequestParameters from .common import _DeletePromptVersionRequestParameters @@ -159,6 +160,7 @@ from .common import _UpdateAgentEngineRequestParameters from .common import _UpdateAgentEngineSessionRequestParameters from .common import _UpdateDatasetParameters +from .common import _UpdateEvaluationExperimentParameters from .common import _UpdateMultimodalDatasetParameters from .common import _UpdateRagConfigRequestParameters from .common import _UpdateRagCorpusRequestParameters @@ -440,6 +442,12 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -1903,6 +1911,9 @@ from .common import UpdateAgentEngineSessionConfig from .common import UpdateAgentEngineSessionConfigDict from .common import UpdateAgentEngineSessionConfigOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict from .common import UpdatePromptConfig from .common import UpdatePromptConfigDict from .common import UpdatePromptConfigOrDict @@ -2173,6 +2184,12 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2371,6 +2388,9 @@ "ListEvaluationMetricsResponse", "ListEvaluationMetricsResponseDict", "ListEvaluationMetricsResponseOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "OptimizeConfig", "OptimizeConfigDict", "OptimizeConfigOrDict", @@ -3720,6 +3740,7 @@ "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", @@ -3732,6 +3753,7 @@ "_GetEvaluationItemParameters", "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", + "_UpdateEvaluationExperimentParameters", "_OptimizeRequestParameters", "_CustomJobParameters", "_GetCustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index cd6fb70b6c..9d9e248f74 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -3017,6 +3017,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel): analysis_configs: Optional[list[AnalysisConfig]] = Field( default=None, description="""""" ) + evaluation_experiment: Optional[str] = Field( + default=None, + description="""The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) class _CreateEvaluationRunParametersDict(TypedDict, total=False): @@ -3046,6 +3052,11 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): analysis_configs: Optional[list[AnalysisConfigDict]] """""" + evaluation_experiment: Optional[str] + """The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + _CreateEvaluationRunParametersOrDict = Union[ _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict @@ -4137,6 +4148,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -6350,6 +6447,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer.""" diff --git a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py index b89d794026..96982e2948 100644 --- a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py +++ b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py @@ -18,6 +18,7 @@ from agentplatform import types from google.genai import types as genai_types import pandas as pd +import contextlib import pytest from unittest import mock import uuid @@ -110,6 +111,40 @@ ) +@pytest.fixture(autouse=True) +def _mock_auto_create_experiment(): + """Stops create_evaluation_run from auto-creating an experiment. + + create_evaluation_run auto-creates a parent EvaluationExperiment when one is + not provided. These replay recordings predate that behavior, so we stub the + experiment creation to return a name-less experiment: no extra HTTP call is + made and the run request omits the evaluationExperiment field, matching the + recordings. + """ + from agentplatform._genai import evals as _evals + + empty = types.EvaluationExperiment() + patchers = [ + mock.patch.object( + _evals.Evals, + "create_evaluation_experiment", + return_value=empty, + ) + ] + if hasattr(_evals.AsyncEvals, "create_evaluation_experiment"): + patchers.append( + mock.patch.object( + _evals.AsyncEvals, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=empty), + ) + ) + with contextlib.ExitStack() as stack: + for patcher in patchers: + stack.enter_context(patcher) + yield + + def test_create_eval_run_data_source_evaluation_set(client): """Tests that create_evaluation_run() creates a correctly structured EvaluationRun.""" client._api_client._http_options.api_version = "v1beta1" diff --git a/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py b/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py new file mode 100644 index 0000000000..2ff500923e --- /dev/null +++ b/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=protected-access,bad-continuation,missing-function-docstring + +from agentplatform._genai import types +from tests.unit.agentplatform.genai.replays import pytest_helper +from google.genai._api_client import HttpOptions +import pytest + +pytestmark = pytest_helper.setup( + file=__file__, + globals_for_file=globals(), + test_method="evals.update_evaluation_experiment", + http_options=HttpOptions( + api_version="v1beta1", + ), +) + +pytest_plugins = ("pytest_asyncio",) + + +def test_update_and_delete(client): + experiment = client.evals.create_evaluation_experiment( + display_name="sdk-update-delete-test" + ) + assert isinstance(experiment, types.EvaluationExperiment) + + updated = client.evals.update_evaluation_experiment( + name=experiment.name, + config={ + "display_name": "sdk-update-delete-test-renamed", + "update_mask": "display_name", + }, + ) + assert isinstance(updated, types.EvaluationExperiment) + assert updated.display_name == "sdk-update-delete-test-renamed" + + delete_operation = client.evals.delete_evaluation_experiment( + name=experiment.name + ) + assert isinstance( + delete_operation, types.DeleteEvaluationExperimentOperation + ) + + +@pytest.mark.asyncio +async def test_update_and_delete_async(client): + experiment = await client.aio.evals.create_evaluation_experiment( + display_name="sdk-update-delete-test-async" + ) + assert isinstance(experiment, types.EvaluationExperiment) + + updated = await client.aio.evals.update_evaluation_experiment( + name=experiment.name, + config={ + "display_name": "sdk-update-delete-test-async-renamed", + "update_mask": "display_name", + }, + ) + assert isinstance(updated, types.EvaluationExperiment) + assert updated.display_name == "sdk-update-delete-test-async-renamed" + + delete_operation = await client.aio.evals.delete_evaluation_experiment( + name=experiment.name + ) + assert isinstance( + delete_operation, types.DeleteEvaluationExperimentOperation + ) diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 6c952203f3..a5a91fed50 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -10595,25 +10595,31 @@ def test_create_evaluation_run_config_default_is_none(self): def test_create_evaluation_run_passes_allow_cross_region_model(self): """Verifies allow_cross_region_model is sent inside evaluationConfig in the API request.""" evals_module = evals.Evals(api_client_=self.mock_api_client) - - evals_module.create_evaluation_run( - dataset=agentplatform_genai_types.EvaluationRunDataSource( - evaluation_set="projects/123/locations/us-central1/evaluationSets/789" - ), - metrics=[ - agentplatform_genai_types.EvaluationRunMetric( - metric="general_quality_v1", - metric_config=agentplatform_genai_types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name="general_quality_v1", - ) - ), - ) - ], - dest="gs://test-bucket/output", - config={"allow_cross_region_model": True}, + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ): + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + config={"allow_cross_region_model": True}, + ) + self.mock_api_client.request.assert_called_once() call_args = self.mock_api_client.request.call_args request_body = call_args[0][2] # Third positional arg is the request dict @@ -10629,25 +10635,33 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) return_value=self.mock_response ) async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) - - await async_evals_module.create_evaluation_run( - dataset=agentplatform_genai_types.EvaluationRunDataSource( - evaluation_set="projects/123/locations/us-central1/evaluationSets/789" - ), - metrics=[ - agentplatform_genai_types.EvaluationRunMetric( - metric="general_quality_v1", - metric_config=agentplatform_genai_types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name="general_quality_v1", - ) - ), - ) - ], - dest="gs://test-bucket/output", - config={"allow_cross_region_model": True}, + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ): + await async_evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + config={"allow_cross_region_model": True}, + ) + self.mock_api_client.async_request.assert_called_once() call_args = self.mock_api_client.async_request.call_args request_body = call_args[0][2] # Third positional arg is the request dict @@ -11886,3 +11900,241 @@ def test_create_evaluation_experiment_passes_all_params(self): assert request_body.get("mergeStrategy") == "SHARED_RESULT_SET" assert request_body.get("labels") == {"team": "agents"} assert request_body.get("metadata") == {"owner": "test"} + + +class TestUpdateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + {"name": self.experiment_name, "displayName": "updated_name"} + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_update_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "updated_name" + + def test_update_evaluation_experiment_uses_patch_and_full_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "patch" + assert call_args[0][1].startswith(self.experiment_name) + + +class TestDeleteEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps({"name": "operations/789"}) + self.mock_api_client.request.return_value = self.mock_response + + def test_delete_evaluation_experiment_uses_delete_and_full_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.delete_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "delete" + assert call_args[0][1] == self.experiment_name + + +class TestCreateEvaluationRunAutoExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationRuns/456", + "displayName": "test_run", + "state": "PENDING", + "evaluationExperiment": ( + "projects/123/locations/us-central1/evaluationExperiments/e1" + ), + } + ) + self.mock_api_client.request.return_value = self.mock_response + self.dataset = agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + self.metrics = [ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ] + + def test_auto_creates_experiment_when_not_provided(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + display_name="my_run", + ) + + mock_create_exp.assert_called_once() + assert mock_create_exp.call_args.kwargs["display_name"] == "my_run" + request_body = self.mock_api_client.request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/e1" + ) + + def test_auto_experiment_default_display_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + evals_module, "create_evaluation_experiment", return_value=experiment + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + ) + + display_name = mock_create_exp.call_args.kwargs["display_name"] + assert display_name.startswith("SDK Experiment ") + + def test_uses_provided_experiment_without_creating(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + with mock.patch.object( + evals_module, "create_evaluation_experiment" + ) as mock_create_exp: + evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + evaluation_experiment=( + "projects/123/locations/us-central1/evaluationExperiments/existing" + ), + ) + + mock_create_exp.assert_not_called() + request_body = self.mock_api_client.request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/existing" + ) + + +class TestAsyncCreateEvaluationRunAutoExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationRuns/456", + "displayName": "test_run", + "state": "PENDING", + "evaluationExperiment": ( + "projects/123/locations/us-central1/evaluationExperiments/e1" + ), + } + ) + self.mock_api_client.async_request = mock.AsyncMock( + return_value=self.mock_response + ) + self.dataset = agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + self.metrics = [ + agentplatform_genai_types.EvaluationRunMetric( + metric="general_quality_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="general_quality_v1", + ) + ), + ) + ] + + @pytest.mark.asyncio + async def test_async_auto_creates_experiment_when_not_provided(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + experiment = agentplatform_genai_types.EvaluationExperiment( + name="projects/123/locations/us-central1/evaluationExperiments/e1" + ) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(return_value=experiment), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + display_name="my_run", + ) + + mock_create_exp.assert_awaited_once() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/e1" + ) + + @pytest.mark.asyncio + async def test_async_uses_provided_experiment_without_creating(self): + async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client) + with mock.patch.object( + async_evals_module, + "create_evaluation_experiment", + new=mock.AsyncMock(), + ) as mock_create_exp: + await async_evals_module.create_evaluation_run( + dataset=self.dataset, + metrics=self.metrics, + dest="gs://test-bucket/output", + evaluation_experiment=( + "projects/123/locations/us-central1/evaluationExperiments/existing" + ), + ) + + mock_create_exp.assert_not_awaited() + request_body = self.mock_api_client.async_request.call_args[0][2] + assert ( + request_body.get("evaluationExperiment") + == "projects/123/locations/us-central1/evaluationExperiments/existing" + ) diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 4d406b7b10..07618dcd78 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -65,6 +65,21 @@ AGENT_DATA = _evals_constant.AGENT_DATA +def _local_timestamp() -> str: + """Returns the current local time as 'M/D/YYYY, H:MM:SS AM/PM'. + + Matches the Agent Platform UI's default experiment name timestamp format + (e.g. '6/1/2026, 1:12:29 PM'). + """ + now = datetime.datetime.now() + hour_12 = now.hour % 12 or 12 + meridiem = "AM" if now.hour < 12 else "PM" + return ( + f"{now.month}/{now.day}/{now.year}, " + f"{hour_12}:{now.minute:02d}:{now.second:02d} {meridiem}" + ) + + @contextlib.contextmanager def _temp_logger_level(logger_name: str, level: int) -> None: # type: ignore[misc] """Temporarily sets the level of a logger.""" diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 9c78c5f04b..37ea26c40d 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -137,6 +137,13 @@ def _CreateEvaluationRunParameters_to_vertex( [item for item in getv(from_object, ["analysis_configs"])], ) + if getv(from_object, ["evaluation_experiment"]) is not None: + setv( + to_object, + ["evaluationExperiment"], + getv(from_object, ["evaluation_experiment"]), + ) + return to_object @@ -1249,6 +1256,7 @@ def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -1263,6 +1271,7 @@ def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -2690,6 +2699,7 @@ def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfoOrDict] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -2837,9 +2847,19 @@ def create_evaluation_run( ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name return self._create_evaluation_run( name=resolved_name, display_name=display_name or resolved_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, @@ -3375,6 +3395,173 @@ def create_evaluation_experiment( self._api_client._verify_response(return_value) return return_value + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("patch", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def get_evaluation_metric( self, *, @@ -3619,6 +3806,7 @@ async def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + evaluation_experiment: Optional[str] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -3633,6 +3821,7 @@ async def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + evaluation_experiment=evaluation_experiment, ) request_url_dict: Optional[dict[str, str]] @@ -4694,6 +4883,36 @@ async def get_evaluation_run( return result + async def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return await self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + async def create_evaluation_run( self, *, @@ -4702,6 +4921,7 @@ async def create_evaluation_run( metrics: list[types.EvaluationRunMetricOrDict], name: Optional[str] = None, display_name: Optional[str] = None, + evaluation_experiment: Optional[str] = None, agent_info: Optional[evals_types.AgentInfo] = None, agent: Optional[str] = None, user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None, @@ -4849,10 +5069,20 @@ async def create_evaluation_run( ) resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent) resolved_name = name or f"evaluation_run_{uuid.uuid4()}" + resolved_experiment = evaluation_experiment + if resolved_experiment is None: + experiment_display_name = ( + display_name or f"SDK Experiment {_evals_common._local_timestamp()}" + ) + created_experiment = await self.create_evaluation_experiment( + display_name=experiment_display_name + ) + resolved_experiment = created_experiment.name result = await self._create_evaluation_run( name=resolved_name, display_name=display_name or resolved_name, + evaluation_experiment=resolved_experiment, data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, @@ -5397,6 +5627,177 @@ async def create_evaluation_experiment( self._api_client._verify_response(return_value) return return_value + async def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "patch", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def get_evaluation_metric( self, *, diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index f692f2351d..a75584f31b 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -374,6 +374,17 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import _DeleteEvaluationExperimentParameters +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict +from .common import _UpdateEvaluationExperimentParameters from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -1693,6 +1704,15 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2810,6 +2830,7 @@ "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", @@ -2884,6 +2905,7 @@ "_GetMultimodalDatasetParameters", "_GetMultimodalDatasetOperationParameters", "_ListMultimodalDatasetsRequestParameters", + "_UpdateEvaluationExperimentParameters", "_UpdateMultimodalDatasetParameters", "_CreateDatasetParameters", "_CreateDatasetVersionParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 3791b6ad13..17d95148d1 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2793,6 +2793,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel): analysis_configs: Optional[list[AnalysisConfig]] = Field( default=None, description="""""" ) + evaluation_experiment: Optional[str] = Field( + default=None, + description="""The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) class _CreateEvaluationRunParametersDict(TypedDict, total=False): @@ -2822,6 +2828,11 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): analysis_configs: Optional[list[AnalysisConfigDict]] """""" + evaluation_experiment: Optional[str] + """The resource name of the parent EvaluationExperiment that this run + belongs to. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + _CreateEvaluationRunParametersOrDict = Union[ _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict @@ -3877,6 +3888,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -5919,6 +6016,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer."""