diff --git a/.env.test b/.env.test index 7679fe7..90c9caa 100644 --- a/.env.test +++ b/.env.test @@ -7,4 +7,5 @@ PYTEST_ADMIN_PASSWORD=start123 PYTEST_DEFAULT_MASTER_IMAGE=python/base PYTEST_ASYNC_MAX_RETRIES=5 PYTEST_ASYNC_RETRY_DELAY_MILLIS=500 -PYTEST_HUB_VERSION=0.12.5 +PYTEST_HUB_VERSION=0.13.0 +PYTEST_RESPONSE_TIMEOUT=5 diff --git a/README.md b/README.md index eab466b..879d69e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPI](https://img.shields.io/pypi/v/flame-hub-client?label=PyPI&cacheSeconds=0)](https://pypi.org/project/flame-hub-client/) -![Code Coverage](https://img.shields.io/badge/Coverage-98%25-brightgreen.svg) +![Code Coverage](https://img.shields.io/badge/Coverage-97%25-brightgreen.svg) ![Python versions](https://img.shields.io/badge/Python->=3.10-blue) [![License](https://img.shields.io/pypi/l/flame-hub-client?label=License&cacheSeconds=0)](https://pypi.org/project/flame-hub-client/) [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-%23FE5196?logo=conventionalcommits&logoColor=white)](https://conventionalcommits.org) @@ -74,22 +74,22 @@ print(my_node.model_dump_json(indent=2)) ```console { - "external_name": null, + "externalName": null, "hidden": false, "name": "my-node", - "realm_id": "794f2375-f043-4789-bd0c-e5534e8deeaa", - "registry_id": null, + "realmId": "794f2375-f043-4789-bd0c-e5534e8deeaa", + "registryId": null, "type": "default", "id": "03636152-e6a8-4e01-994e-18b2b0c3a935", - "public_key": null, + "publicKey": null, "online": false, "registry": null, - "registry_project_id": null, - "registry_project": null, - "robot_id": null, - "client_id": "2d3e19b4-6708-4279-b2a7-34ad42638e4b", - "created_at": "2025-05-19T15:43:57.859000Z", - "updated_at": "2025-05-19T15:43:57.859000Z" + "registryProjectId": null, + "registryProject": null, + "robotId": null, + "clientId": "2d3e19b4-6708-4279-b2a7-34ad42638e4b", + "createdAt": "2025-05-19T15:43:57.859000Z", + "updatedAt": "2025-05-19T15:43:57.859000Z" } ``` diff --git a/docs/clients_api.rst b/docs/clients_api.rst index e7acfc9..323ca0e 100644 --- a/docs/clients_api.rst +++ b/docs/clients_api.rst @@ -4,11 +4,12 @@ Clients .. autoclass:: flame_hub._base_client.BaseClient :private-members: _get_all_resources, _find_all_resources, _create_resource, _get_single_resource, _update_resource, - _delete_resource + _delete_resource, _unwrap_single_resource .. autoclass:: flame_hub.AuthClient :members: :undoc-members: + :private-members: _unwrap_single_resource .. autoclass:: flame_hub.CoreClient :members: diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 6d2dfc6..5d90d5e 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -190,8 +190,8 @@ left unset, the client will sort in ascending order by default. .. code-block:: python - nodes = core_client.find_nodes(sort={"by": "created_at"}) - sedon = core_client.find_nodes(sort={"by": "created_at", "order": "descending"}) + nodes = core_client.find_nodes(sort={"by": "createdAt"}) + sedon = core_client.find_nodes(sort={"by": "createdAt", "order": "descending"}) assert nodes == sedon[::-1] @@ -322,7 +322,7 @@ It is also possible to retrieve all names of includable properties for a specifi .. code-block:: - ('registry', 'registry_project') + ('registry', 'registryProject') Overriding authentication per request @@ -379,11 +379,11 @@ possible, including status code and additional information in the response body. .. code-block:: console - received status code 400 (undefined): Can't find realm entity by realm_id + received status code 400 (undefined): Can't find realm entity by realmId { "status_code": 400, "code": "undefined", - "message": "Can't find realm entity by realm_id" + "message": "Can't find realm entity by realmId" } In this example a :py:exc:`.HubAPIError` is raised because there is no realm with an ID that matches the dynamically @@ -413,7 +413,7 @@ Hub instance. { "hidden": false, - "external_name": null + "externalName": null } Check out all implemented models :doc:`here `. diff --git a/flame_hub/_auth_client.py b/flame_hub/_auth_client.py index 76c4670..2a744b2 100644 --- a/flame_hub/_auth_client.py +++ b/flame_hub/_auth_client.py @@ -3,8 +3,7 @@ import typing as t import typing_extensions as te -from pydantic import BaseModel, Field, WrapValidator, EmailStr, ConfigDict -from pydantic.alias_generators import to_camel +from pydantic import Field, WrapValidator, EmailStr from flame_hub._base_client import ( BaseClient, @@ -20,17 +19,14 @@ ResourceListResult, AuthParam, BaseKwargs, + ConfigBaseModel, + SingleResourceResult, ) from flame_hub._defaults import DEFAULT_AUTH_BASE_URL -class AuthBaseModel(BaseModel): - model_config = ConfigDict( - alias_generator=to_camel, - validate_by_alias=True, - validate_by_name=True, - serialize_by_alias=True, - ) +class AuthBaseModel(ConfigBaseModel): + pass class CreateRealm(AuthBaseModel): @@ -283,7 +279,11 @@ def create_realm( def delete_realm(self, realm_id: Realm | uuid.UUID | str, **params: te.Unpack[BaseKwargs]): self._delete_resource("realms", realm_id, **params) - def get_realm(self, realm_id: Realm | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Realm | None: + def get_realm( + self, + realm_id: Realm | uuid.UUID | str, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Realm]: return self._get_single_resource(Realm, "realms", realm_id, **params) def update_realm( @@ -329,7 +329,7 @@ def create_permission( def get_permission( self, permission_id: Permission | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> Permission | None: + ) -> SingleResourceResult[Permission]: return self._get_single_resource( Permission, "permissions", permission_id, include=get_includable_names(Permission), **params ) @@ -374,7 +374,7 @@ def create_role( **params, ) - def get_role(self, role_id: Role | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Role | None: + def get_role(self, role_id: Role | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> SingleResourceResult[Role]: return self._get_single_resource(Role, "roles", role_id, include=get_includable_names(Role), **params) def delete_role(self, role_id: Role | uuid.UUID | str, **params: te.Unpack[BaseKwargs]): @@ -417,7 +417,7 @@ def create_role_permission( def get_role_permission( self, role_permission_id: RolePermission | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> RolePermission | None: + ) -> SingleResourceResult[RolePermission]: return self._get_single_resource( RolePermission, "role-permissions", @@ -475,7 +475,7 @@ def create_user( **params, ) - def get_user(self, user_id: User | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> User | None: + def get_user(self, user_id: User | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> SingleResourceResult[User]: return self._get_single_resource(User, "users", user_id, include=get_includable_names(User), **params) def delete_user(self, user_id: User | uuid.UUID | str, **params: te.Unpack[BaseKwargs]): @@ -530,7 +530,7 @@ def create_user_permission( def get_user_permission( self, user_permission_id: UserPermission | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> UserPermission | None: + ) -> SingleResourceResult[UserPermission]: return self._get_single_resource( UserPermission, "user-permissions", @@ -577,7 +577,7 @@ def create_user_role( def get_user_role( self, user_role_id: UserRole | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> UserRole | None: + ) -> SingleResourceResult[UserRole]: return self._get_single_resource( UserRole, "user-roles", user_role_id, include=get_includable_names(UserRole), **params ) @@ -630,7 +630,11 @@ def create_client( def delete_client(self, client_id: Client | uuid.UUID | str, **params: te.Unpack[BaseKwargs]): self._delete_resource("clients", client_id, **params) - def get_client(self, client_id: Client | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Client | None: + def get_client( + self, + client_id: Client | uuid.UUID | str, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Client]: return self._get_single_resource(Client, "clients", client_id, include=get_includable_names(Client), **params) def get_clients(self, **params: te.Unpack[GetKwargs]) -> ResourceListResult[Client]: diff --git a/flame_hub/_base_client.py b/flame_hub/_base_client.py index 65748b8..834c196 100644 --- a/flame_hub/_base_client.py +++ b/flame_hub/_base_client.py @@ -6,7 +6,8 @@ import httpx2 as httpx import typing_extensions as te -from pydantic import BaseModel, ValidatorFunctionWrapHandler, ValidationError, ConfigDict +from pydantic import BaseModel, ValidatorFunctionWrapHandler, ValidationError, ConfigDict, Field +from pydantic.alias_generators import to_camel from flame_hub._exceptions import new_hub_api_error_from_response, HubAPIError from flame_hub._auth_flows import PasswordAuth, ClientAuth, StaticAuth @@ -19,6 +20,17 @@ class UNSET(BaseModel): UNSET_T = type[UNSET] +class ConfigBaseModel(BaseModel): + """Base model that defines all configurations that are inherited to all model classes.""" + + model_config = ConfigDict( + alias_generator=to_camel, + validate_by_alias=True, + validate_by_name=True, + serialize_by_alias=True, + ) + + ResourceT = t.TypeVar("ResourceT", bound=BaseModel) """Base resource type which assumes :py:class:`~pydantic.BaseModel` as the base class.""" @@ -90,26 +102,45 @@ def uuid_validator(value: t.Any, handler: ValidatorFunctionWrapHandler) -> uuid. raise e -class ResourceListMeta(BaseModel): +class SingleResourceMeta(BaseModel): + """Available meta info for single resource responses. + + See Also + -------- + :py:meth:`._get_single_resource` + """ + + model_config = ConfigDict( + validate_by_alias=True, + validate_by_name=True, + ) + + # Otherwise the "schema" attribute from BaseModel will be overshadowed. + response_schema: dict = Field(alias="schema") + + +class ResourceListMeta(SingleResourceMeta): """Resource for meta information on list responses. See Also -------- - :py:meth:`._get_all_resources`, :py:meth:`._find_all_resources` + :py:meth:`._get_all_resources`, :py:meth:`._find_all_resources`, :py:class:`.SingleResourceMeta` """ - model_config = ConfigDict(extra="allow") - """Based on the type of the request, additional attributes like ``limit`` or ``offset`` may be available.""" total: int """The total amount of records of a specific resource type.""" + limit: int + """Amount of returned resources.""" + offset: int + """Amount of resources to skip before resources are added to the returned list.""" class ResourceList(BaseModel, t.Generic[ResourceT]): - """Resource for list responses. + """Model for list responses. See Also -------- - :py:meth:`._get_all_resources`, :py:meth:`._find_all_resources` + :py:meth:`._get_all_resources`, :py:meth:`._find_all_resources`, :py:class:`ResourceListMeta` """ data: list[ResourceT] @@ -118,8 +149,24 @@ class ResourceList(BaseModel, t.Generic[ResourceT]): """Attribute which holds meta information about the result and the requested resource type.""" -# Generic type alias for all get and find methods. +class WrappedResource(BaseModel, t.Generic[ResourceT]): + """Model for wrapped singular resources. + + See Also + -------- + :py:meth:`._get_singular_resource`, :py:class:`.SingleResourceMeta` + """ + + data: ResourceT + """Attribute which holds the retrieved resource.""" + meta: SingleResourceMeta + """Attribute which holds meta information about the requested resource.""" + + +# Generic type alias for all list get and find methods. ResourceListResult: t.TypeAlias = list[ResourceT] | tuple[list[ResourceT], ResourceListMeta] +# Generic type alias for all get methods. +SingleResourceResult: t.TypeAlias = ResourceT | tuple[ResourceT, SingleResourceMeta] | None class SortParams(te.TypedDict, total=False): @@ -365,7 +412,7 @@ def build_include_params(include_params: IncludeParams | None = None) -> dict: include_params = (include_params,) # coalesce into tuple # unravel iterable and merge into tuple - include_params = tuple(p for p in include_params) + include_params = tuple(to_camel(p) for p in include_params) if len(include_params) == 0: return {} @@ -381,7 +428,7 @@ def build_field_params(field_params: FieldParams | None = None) -> dict: field_params = (field_params,) # coalesce into tuple # unravel iterable and merge into tuple - field_params = tuple(p for p in field_params) + field_params = tuple(to_camel(p) for p in field_params) # only allow the addition of fields field_params = tuple(f"+{p}" for p in field_params) @@ -413,6 +460,14 @@ def resolve_auth(auth: AuthParam) -> ClientAuth | PasswordAuth | StaticAuth | No return auth +def _is_enveloped(response_body: dict) -> bool: + """Checks if a response body is enveloped. In that case resources are available via the 'data' key and further meta + information via the 'meta' key.""" + if "data" in response_body and "meta" in response_body: + return True + return False + + class BaseClient(object): """The base class for other client classes. @@ -637,6 +692,9 @@ def _create_resource( r = self._request("POST", *path, expected_code=expected_code, json=resource.model_dump(mode="json"), **params) + if _is_enveloped(r.json()): + # The meta field is empty for create responses so it gets thrown away here. + return resource_type(**r.json()["data"]) return resource_type(**r.json()) def _get_single_resource( @@ -646,7 +704,7 @@ def _get_single_resource( include: IncludeParams | None = None, expected_code: int = httpx.codes.OK.value, **params: te.Unpack[GetKwargs], - ) -> ResourceT | None: + ) -> SingleResourceResult: """Get a single resource of a certain type at the specified path. This method accesses the endpoint ``*path`` and returns the resource of type ``resource_type``. In contrast to @@ -684,17 +742,17 @@ def _get_single_resource( If the status code of the response does not match ``expected_code`` or 404. :py:exc:`~pydantic_core._pydantic_core.ValidationError` If the resource returned by the Hub instance does not validate with the given ``resource_type``. + :py:exc:`ValueError` + If ``meta=True``, but the endpoint does not send meta information in the response body for that specific + resource. See Also -------- :py:meth:`._get_all_resources`, :py:meth:`._find_all_resources` - - Notes - ----- - ``meta`` has no relevance for this method. """ field_params = params.pop("fields", None) + meta_flag = params.pop("meta", False) request_params = build_field_params(field_params) | build_include_params(include) @@ -706,7 +764,15 @@ def _get_single_resource( else: raise - return resource_type(**r.json()) + if _is_enveloped(r.json()): + wrapped_resource = WrappedResource[resource_type](**r.json()) + if meta_flag: + return wrapped_resource.data, wrapped_resource.meta + return wrapped_resource.data + else: + if meta_flag: + raise ValueError(f"Single resources of type {resource_type} do not have meta data.") + return resource_type(**r.json()) def _update_resource( self, @@ -760,6 +826,9 @@ def _update_resource( **params, ) + if _is_enveloped(r.json()): + # The meta field is empty for update responses so it gets thrown away here. + return resource_type(**r.json()["data"]) return resource_type(**r.json()) def _delete_resource( diff --git a/flame_hub/_core_client.py b/flame_hub/_core_client.py index 55164b1..92b7dda 100644 --- a/flame_hub/_core_client.py +++ b/flame_hub/_core_client.py @@ -5,7 +5,7 @@ import httpx2 as httpx import typing_extensions as te -from pydantic import BaseModel, WrapValidator, Field, BeforeValidator +from pydantic import WrapValidator, Field, BeforeValidator from flame_hub._auth_client import Realm from flame_hub._base_client import ( @@ -24,14 +24,21 @@ ResourceListResult, AuthParam, BaseKwargs, + ConfigBaseModel, + SingleResourceResult, ) from flame_hub._defaults import DEFAULT_CORE_BASE_URL from flame_hub._storage_client import Bucket, BucketFile + +class CoreBaseModel(ConfigBaseModel): + pass + + RegistryCommand = t.Literal["setup", "cleanup"] -class CreateRegistry(BaseModel): +class CreateRegistry(CoreBaseModel): name: str host: str account_name: str | None @@ -44,7 +51,7 @@ class Registry(CreateRegistry): updated_at: datetime -class UpdateRegistry(BaseModel): +class UpdateRegistry(CoreBaseModel): name: str | UNSET_T = UNSET host: str | UNSET_T = UNSET account_name: str | None | UNSET_T = UNSET @@ -54,7 +61,7 @@ class UpdateRegistry(BaseModel): RegistryProjectType = t.Literal["default", "aggregator", "incoming", "outgoing", "masterImages", "node"] -class CreateRegistryProject(BaseModel): +class CreateRegistryProject(CoreBaseModel): name: str type: RegistryProjectType registry_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] @@ -76,7 +83,7 @@ class RegistryProject(CreateRegistryProject): updated_at: datetime -class UpdateRegistryProject(BaseModel): +class UpdateRegistryProject(CoreBaseModel): name: str | UNSET_T = UNSET type: RegistryProjectType | UNSET_T = UNSET registry_id: t.Annotated[uuid.UUID | UNSET_T, Field(), WrapValidator(uuid_validator)] = UNSET @@ -88,7 +95,7 @@ class UpdateRegistryProject(BaseModel): NodeType = t.Literal["aggregator", "default"] -class CreateNode(BaseModel): +class CreateNode(CoreBaseModel): external_name: str | None hidden: bool | None name: str @@ -109,7 +116,7 @@ class Node(CreateNode): updated_at: datetime -class UpdateNode(BaseModel): +class UpdateNode(CoreBaseModel): hidden: bool | UNSET_T = UNSET external_name: str | None | UNSET_T = UNSET type: NodeType | UNSET_T = UNSET @@ -118,27 +125,27 @@ class UpdateNode(BaseModel): registry_id: t.Annotated[uuid.UUID | None | UNSET_T, Field(), WrapValidator(uuid_validator)] = UNSET -class NodeRegistryCredentials(BaseModel): +class NodeRegistryCredentials(CoreBaseModel): host: str external_name: str account_name: str | None account_secret: str | None -class ClientCredentials(BaseModel): +class ClientCredentials(CoreBaseModel): id: uuid.UUID secret: str | None name: str display_name: str -class UpdateClientCredentials(BaseModel): +class UpdateClientCredentials(CoreBaseModel): secret: str | None | UNSET_T = UNSET name: str | UNSET_T = UNSET display_name: str | UNSET_T = UNSET -class MasterImageGroup(BaseModel): +class MasterImageGroup(CoreBaseModel): id: uuid.UUID name: str path: str @@ -170,7 +177,7 @@ def ensure_position_none(value: t.Any) -> t.Any: ProcessStatus = t.Literal["starting", "started", "stopping", "stopped", "executing", "executed", "failed"] -class MasterImage(BaseModel): +class MasterImage(CoreBaseModel): id: uuid.UUID path: str | None virtual_path: str @@ -184,7 +191,7 @@ class MasterImage(BaseModel): updated_at: datetime -class CreateProject(BaseModel): +class CreateProject(CoreBaseModel): description: str | None master_image_id: t.Annotated[uuid.UUID | None, Field(), WrapValidator(uuid_validator)] name: str @@ -202,7 +209,7 @@ class Project(CreateProject): user_id: uuid.UUID | None -class UpdateProject(BaseModel): +class UpdateProject(CoreBaseModel): description: str | None | UNSET_T = UNSET master_image_id: t.Annotated[uuid.UUID | None | UNSET_T, Field(), WrapValidator(uuid_validator)] = UNSET name: str | UNSET_T = UNSET @@ -212,7 +219,7 @@ class UpdateProject(BaseModel): ProjectNodeApprovalStatus = t.Literal["rejected", "approved"] -class CreateProjectNode(BaseModel): +class CreateProjectNode(CoreBaseModel): node_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] project_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] @@ -229,7 +236,7 @@ class ProjectNode(CreateProjectNode): node_realm_id: uuid.UUID -class UpdateProjectNode(BaseModel): +class UpdateProjectNode(CoreBaseModel): comment: str | None | UNSET_T = UNSET approval_status: ProjectNodeApprovalStatus | None | UNSET_T = UNSET @@ -238,7 +245,7 @@ class UpdateProjectNode(BaseModel): LogChannel = t.Literal["http", "websocket", "background", "system"] -class Log(BaseModel): +class Log(CoreBaseModel): time: str message: str service: str @@ -247,7 +254,7 @@ class Log(BaseModel): labels: dict[str, str | None] -class CreateAnalysis(BaseModel): +class CreateAnalysis(CoreBaseModel): description: str | None name: str | None display_name: str | None @@ -292,7 +299,7 @@ class Analysis(CreateAnalysis): master_image: t.Annotated[MasterImage | None, IsIncludable] = None -class UpdateAnalysis(BaseModel): +class UpdateAnalysis(CoreBaseModel): description: str | None | UNSET_T = UNSET name: str | UNSET_T = UNSET display_name: str | None | UNSET_T = UNSET @@ -317,7 +324,7 @@ class UpdateAnalysis(BaseModel): ] -class CreateAnalysisNode(BaseModel): +class CreateAnalysisNode(CoreBaseModel): analysis_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] node_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] @@ -341,14 +348,14 @@ class AnalysisNode(CreateAnalysisNode): node_realm_id: uuid.UUID -class UpdateAnalysisNode(BaseModel): +class UpdateAnalysisNode(CoreBaseModel): comment: str | None | UNSET_T = UNSET approval_status: AnalysisNodeApprovalStatus | None | UNSET_T = UNSET execution_status: ProcessStatus | None | UNSET_T = UNSET execution_progress: int | None | UNSET_T = UNSET -class CreateAnalysisNodeLog(BaseModel): +class CreateAnalysisNodeLog(CoreBaseModel): analysis_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] node_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] code: str | None @@ -363,7 +370,7 @@ class AnalysisBucketType(str, Enum): TEMP = "TEMP" -class CreateAnalysisBucket(BaseModel): +class CreateAnalysisBucket(CoreBaseModel): type: AnalysisBucketType bucket_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] analysis_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] @@ -377,7 +384,7 @@ class AnalysisBucket(CreateAnalysisBucket): realm_id: uuid.UUID -class CreateAnalysisBucketFile(BaseModel): +class CreateAnalysisBucketFile(CoreBaseModel): path: str bucket_file_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] bucket_id: t.Annotated[uuid.UUID, Field(), WrapValidator(uuid_validator)] @@ -397,7 +404,7 @@ class AnalysisBucketFile(CreateAnalysisBucketFile): analysis: t.Annotated[Analysis, IsIncludable] = None -class UpdateAnalysisBucketFile(BaseModel): +class UpdateAnalysisBucketFile(CoreBaseModel): root: bool | UNSET_T = UNSET @@ -450,7 +457,7 @@ def create_node( **params, ) - def get_node(self, node_id: Node | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Node | None: + def get_node(self, node_id: Node | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> SingleResourceResult[Node]: return self._get_single_resource(Node, "nodes", node_id, include=get_includable_names(Node), **params) def delete_node( @@ -490,7 +497,7 @@ def get_node_registry_credentials( self, node_id: Node | uuid.UUID | str, **params: te.Unpack[GetKwargs], - ) -> NodeRegistryCredentials | None: + ) -> SingleResourceResult[NodeRegistryCredentials]: """Returns the node's registry project credentials.""" return self._get_single_resource( @@ -506,7 +513,7 @@ def get_node_client_credentials( self, node_id: Node | uuid.UUID | str, **params: te.Unpack[GetKwargs], - ) -> ClientCredentials | None: + ) -> SingleResourceResult[ClientCredentials]: """Returns the node's client credentials.""" return self._get_single_resource( @@ -549,7 +556,7 @@ def get_master_image_groups(self, **params: te.Unpack[GetKwargs]) -> ResourceLis def get_master_image_group( self, master_image_group_id: MasterImageGroup | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> MasterImageGroup | None: + ) -> SingleResourceResult[MasterImageGroup]: return self._get_single_resource(MasterImageGroup, "master-image-groups", master_image_group_id, **params) def find_master_image_groups(self, **params: te.Unpack[FindAllKwargs]) -> ResourceListResult[MasterImageGroup]: @@ -560,7 +567,7 @@ def get_master_images(self, **params: te.Unpack[GetKwargs]) -> ResourceListResul def get_master_image( self, master_image_id: MasterImage | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> MasterImage | None: + ) -> SingleResourceResult[MasterImage]: return self._get_single_resource(MasterImage, "master-images", master_image_id, **params) def find_master_images(self, **params: te.Unpack[FindAllKwargs]) -> ResourceListResult[MasterImage]: @@ -623,7 +630,11 @@ def create_project( def delete_project(self, project_id: Project | uuid.UUID | str, **params: te.Unpack[BaseKwargs]): self._delete_resource("projects", project_id, **params) - def get_project(self, project_id: Project | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Project | None: + def get_project( + self, + project_id: Project | uuid.UUID | str, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Project]: return self._get_single_resource( Project, "projects", project_id, include=get_includable_names(Project), **params ) @@ -672,7 +683,7 @@ def find_project_nodes(self, **params: te.Unpack[FindAllKwargs]) -> ResourceList def get_project_node( self, project_node_id: ProjectNode | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> ProjectNode | None: + ) -> SingleResourceResult[ProjectNode]: return self._get_single_resource( ProjectNode, "project-nodes", project_node_id, include=get_includable_names(ProjectNode), **params ) @@ -727,7 +738,11 @@ def get_analyses(self, **params: te.Unpack[GetKwargs]) -> ResourceListResult[Ana def find_analyses(self, **params: te.Unpack[FindAllKwargs]) -> ResourceListResult[Analysis]: return self._find_all_resources(Analysis, "analyses", include=get_includable_names(Analysis), **params) - def get_analysis(self, analysis_id: Analysis | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Analysis | None: + def get_analysis( + self, + analysis_id: Analysis | uuid.UUID | str, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Analysis]: return self._get_single_resource( Analysis, "analyses", analysis_id, include=get_includable_names(Analysis), **params ) @@ -772,13 +787,13 @@ def send_analysis_command( **params, ) - return Analysis(**r.json()) + return Analysis(**r.json()["data"]) def get_analysis_client_credentials( self, analysis_id: Analysis | uuid.UUID | str, **params: te.Unpack[GetKwargs], - ) -> ClientCredentials | None: + ) -> SingleResourceResult[ClientCredentials]: """Returns the client credentials of the analysis.""" return self._get_single_resource( @@ -853,7 +868,7 @@ def update_analysis_node( def get_analysis_node( self, analysis_node_id: AnalysisNode | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> AnalysisNode | None: + ) -> SingleResourceResult[AnalysisNode]: return self._get_single_resource( AnalysisNode, "analysis-nodes", analysis_node_id, include=get_includable_names(AnalysisNode), **params ) @@ -952,7 +967,7 @@ def find_analysis_buckets(self, **params: te.Unpack[FindAllKwargs]) -> ResourceL def get_analysis_bucket( self, analysis_bucket_id: AnalysisBucket | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> AnalysisBucket | None: + ) -> SingleResourceResult[AnalysisBucket]: return self._get_single_resource( AnalysisBucket, "analysis-buckets", @@ -973,7 +988,7 @@ def find_analysis_bucket_files(self, **params: te.Unpack[FindAllKwargs]) -> Reso def get_analysis_bucket_file( self, analysis_bucket_file_id: AnalysisBucketFile | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> AnalysisBucketFile | None: + ) -> SingleResourceResult[AnalysisBucketFile]: return self._get_single_resource( AnalysisBucketFile, "analysis-bucket-files", @@ -1040,7 +1055,11 @@ def create_registry( **params, ) - def get_registry(self, registry_id: Registry | uuid.UUID | str, **params: te.Unpack[GetKwargs]) -> Registry | None: + def get_registry( + self, + registry_id: Registry | uuid.UUID | str, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Registry]: return self._get_single_resource(Registry, "registries", registry_id, **params) def delete_registry( @@ -1115,7 +1134,7 @@ def create_registry_project( def get_registry_project( self, registry_project_id: RegistryProject | uuid.UUID | str, **params: te.Unpack[GetKwargs] - ) -> RegistryProject | None: + ) -> SingleResourceResult[RegistryProject]: return self._get_single_resource( RegistryProject, "registry-projects", diff --git a/flame_hub/_storage_client.py b/flame_hub/_storage_client.py index 4cc18f2..2605250 100644 --- a/flame_hub/_storage_client.py +++ b/flame_hub/_storage_client.py @@ -4,11 +4,9 @@ import httpx2 as httpx import typing_extensions as te -from pydantic import BaseModel from flame_hub._base_client import ( BaseClient, - ResourceList, obtain_uuid_from, FindAllKwargs, GetKwargs, @@ -18,11 +16,17 @@ ResourceListResult, AuthParam, BaseKwargs, + ConfigBaseModel, + SingleResourceResult, ) from flame_hub._defaults import DEFAULT_STORAGE_BASE_URL -class CreateBucket(BaseModel): +class StorageBaseModel(ConfigBaseModel): + pass + + +class CreateBucket(StorageBaseModel): name: str region: str | None @@ -36,7 +40,7 @@ class Bucket(CreateBucket): realm_id: uuid.UUID | None -class BucketFile(BaseModel): +class BucketFile(StorageBaseModel): id: uuid.UUID name: str path: str @@ -100,7 +104,11 @@ def get_buckets(self, **params: te.Unpack[GetKwargs]) -> ResourceListResult[Buck def find_buckets(self, **params: te.Unpack[FindAllKwargs]) -> ResourceListResult[Bucket]: return self._find_all_resources(Bucket, "buckets", **params) - def get_bucket(self, bucket_id: Bucket | str | uuid.UUID, **params: te.Unpack[GetKwargs]) -> Bucket | None: + def get_bucket( + self, + bucket_id: Bucket | str | uuid.UUID, + **params: te.Unpack[GetKwargs], + ) -> SingleResourceResult[Bucket]: return self._get_single_resource(Bucket, "buckets", bucket_id, **params) def stream_bucket_tarball( @@ -146,14 +154,14 @@ def upload_to_bucket( **params, ) - return ResourceList[BucketFile](**r.json()).data + return [BucketFile(**d) for d in r.json()["data"]] def delete_bucket_file(self, bucket_file_id: BucketFile | str | uuid.UUID, **params: te.Unpack[BaseKwargs]): self._delete_resource("bucket-files", bucket_file_id, **params) def get_bucket_file( self, bucket_file_id: BucketFile | str | uuid.UUID, **params: te.Unpack[GetKwargs] - ) -> BucketFile | None: + ) -> SingleResourceResult[BucketFile]: return self._get_single_resource( BucketFile, "bucket-files", bucket_file_id, include=get_includable_names(BucketFile), **params ) diff --git a/flame_hub/models.py b/flame_hub/models.py index 27dd379..8378f80 100644 --- a/flame_hub/models.py +++ b/flame_hub/models.py @@ -57,7 +57,6 @@ "IsOptionalField", "IsIncludable", "ResourceListMeta", - "ResourceList", "AccessToken", "RefreshToken", "CreateClient", @@ -66,9 +65,17 @@ "NodeRegistryCredentials", "ClientCredentials", "UpdateClientCredentials", + "ConfigBaseModel", + "AuthBaseModel", + "CoreBaseModel", + "StorageBaseModel", + "SingleResourceMeta", + "ResourceList", + "WrappedResource", ] from ._core_client import ( + CoreBaseModel, CreateNode, Node, NodeType, @@ -106,9 +113,19 @@ ClientCredentials, UpdateClientCredentials, ) -from ._storage_client import CreateBucket, Bucket, BucketFile -from ._base_client import UNSET, IsOptionalField, IsIncludable, ResourceListMeta, ResourceList +from ._storage_client import CreateBucket, Bucket, BucketFile, StorageBaseModel +from ._base_client import ( + UNSET, + IsOptionalField, + IsIncludable, + SingleResourceMeta, + ResourceListMeta, + ConfigBaseModel, + ResourceList, + WrappedResource, +) from ._auth_client import ( + AuthBaseModel, CreateRealm, UpdateRealm, Realm, diff --git a/flame_hub/types.py b/flame_hub/types.py index e6cbe43..80715a0 100644 --- a/flame_hub/types.py +++ b/flame_hub/types.py @@ -23,6 +23,7 @@ "ProcessStatus", "LogChannel", "ReadableBinary", + "SingleResourceResult", "ResourceListResult", "AuthParam", "BaseKwargs", @@ -40,6 +41,7 @@ GetKwargs, ResourceT, UNSET_T, + SingleResourceResult, ResourceListResult, AuthParam, BaseKwargs, diff --git a/tests/conftest.py b/tests/conftest.py index 28d4135..28368c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,11 @@ def use_testcontainers() -> bool: return os.getenv("PYTEST_USE_TESTCONTAINERS", "1").strip().lower() not in ("0", "false", "n", "no", "") +@pytest.fixture(scope="session") +def response_timeout() -> int: + return int(os.getenv("PYTEST_RESPONSE_TIMEOUT", "5")) + + @pytest.fixture(scope="session") def hub_version() -> str: return os.getenv("PYTEST_HUB_VERSION", "0.8.13") @@ -339,18 +344,18 @@ def password_auth(nginx, auth_base_url, auth_admin_username, auth_admin_password @pytest.fixture(scope="session") -def auth_client(password_auth, auth_base_url): - yield AuthClient(base_url=auth_base_url, auth=password_auth) +def auth_client(password_auth, auth_base_url, response_timeout): + yield AuthClient(client=httpx.Client(base_url=auth_base_url, auth=password_auth, timeout=response_timeout)) @pytest.fixture(scope="session") -def core_client(password_auth, core_base_url): - yield CoreClient(base_url=core_base_url, auth=password_auth) +def core_client(password_auth, core_base_url, response_timeout): + yield CoreClient(client=httpx.Client(base_url=core_base_url, auth=password_auth, timeout=response_timeout)) @pytest.fixture(scope="session") -def storage_client(password_auth, storage_base_url): - yield StorageClient(base_url=storage_base_url, auth=password_auth) +def storage_client(password_auth, storage_base_url, response_timeout): + yield StorageClient(client=httpx.Client(base_url=storage_base_url, auth=password_auth, timeout=response_timeout)) @pytest.fixture(scope="session") diff --git a/tests/test_auth.py b/tests/test_auth.py index a266b82..e2e74ab 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -229,7 +229,7 @@ def test_get_role_permissions(auth_client, role_permission, role_permission_incl @pytest.mark.xfail(reason="bug in authup") def test_find_role_permissions(auth_client, role_permission, role_permission_includables): # Use "role_id" for filtering because there is no filter mechanism for attribute "id". - role_perms_find = auth_client.find_role_permissions(filter={"role_id": role_permission.role_id}) + role_perms_find = auth_client.find_role_permissions(filter={"roleId": role_permission.role_id}) assert [role_permission.id] == [rp.id for rp in role_perms_find] assert all( @@ -296,7 +296,7 @@ def test_get_user_permissions(auth_client, user_permission, user_permission_incl @pytest.mark.xfail(reason="bug in authup") def test_find_user_permissions(auth_client, user_permission, user_permission_includables): # Use "user_id" for filtering because there is no filter mechanism for attribute "id". - user_perms_find = auth_client.find_user_permissions(filter={"user_id": user_permission.user_id}) + user_perms_find = auth_client.find_user_permissions(filter={"userId": user_permission.user_id}) assert len(user_perms_find) > 0 assert all( @@ -327,7 +327,7 @@ def test_get_user_roles(auth_client, user_role, user_role_includables): @pytest.mark.xfail(reason="bug in authup") def test_find_user_roles(auth_client, user_role, user_role_includables): # Use "user_id" for filtering because there is no filter mechanism for attribute "id". - user_roles_find = auth_client.find_user_roles(filter={"user_id": user_role.user_id}) + user_roles_find = auth_client.find_user_roles(filter={"userId": user_role.user_id}) assert len(user_roles_find) > 0 assert all(includable in ur.model_fields_set for ur in user_roles_find for includable in user_role_includables) diff --git a/tests/test_core.py b/tests/test_core.py index f604259..b2bcc47 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -37,30 +37,30 @@ def sync_master_images(core_client): def master_image(core_client): default_master_image = os.getenv("PYTEST_DEFAULT_MASTER_IMAGE", "python/base") - if len(core_client.find_master_images(filter={"virtual_path": default_master_image})) != 1: + if len(core_client.find_master_images(filter={"virtualPath": default_master_image})) != 1: sync_master_images(core_client) def _check_default_master_image_available(): - assert len(core_client.find_master_images(filter={"virtual_path": default_master_image})) == 1 + assert len(core_client.find_master_images(filter={"virtualPath": default_master_image})) == 1 assert_eventually(_check_default_master_image_available, max_retries=10, delay_millis=1000) - return core_client.find_master_images(filter={"virtual_path": default_master_image})[0] + return core_client.find_master_images(filter={"virtualPath": default_master_image})[0] @pytest.fixture(scope="module") def master_image_group(core_client, master_image): - if len(core_client.find_master_image_groups(filter={"virtual_path": master_image.group_virtual_path})) != 1: + if len(core_client.find_master_image_groups(filter={"virtualPath": master_image.group_virtual_path})) != 1: sync_master_images(core_client) def _check_default_master_image_group_available(): assert ( - len(core_client.find_master_image_groups(filter={"virtual_path": master_image.group_virtual_path})) == 1 + len(core_client.find_master_image_groups(filter={"virtualPath": master_image.group_virtual_path})) == 1 ) assert_eventually(_check_default_master_image_group_available, max_retries=10, delay_millis=1000) - return core_client.find_master_image_groups(filter={"virtual_path": master_image.group_virtual_path})[0] + return core_client.find_master_image_groups(filter={"virtualPath": master_image.group_virtual_path})[0] @pytest.fixture() @@ -129,11 +129,11 @@ def analysis_node_includables(): @pytest.fixture() def analysis_code_bucket(core_client, analysis): def _wait_for_buckets(): - assert len(core_client.find_analysis_buckets(filter={"analysis_id": analysis.id})) != 0 + assert len(core_client.find_analysis_buckets(filter={"analysisId": analysis.id})) != 0 assert_eventually(_wait_for_buckets) - analysis_buckets = core_client.find_analysis_buckets(filter={"analysis_id": analysis.id}) + analysis_buckets = core_client.find_analysis_buckets(filter={"analysisId": analysis.id}) code_buckets = [bucket for bucket in analysis_buckets if bucket.type == AnalysisBucketType.CODE] assert len(code_buckets) == 1 @@ -163,14 +163,14 @@ def analysis_bucket_file(core_client, storage_client, analysis_code_bucket, rng_ def _wait_for_analysis_bucket_file(): analysis_bucket_files = core_client.find_analysis_bucket_files( - filter={"analysis_id": analysis_code_bucket.analysis_id} + filter={"analysisId": analysis_code_bucket.analysis_id} ) assert len(analysis_bucket_files) == 1 assert_eventually(_wait_for_analysis_bucket_file) analysis_bucket_file = core_client.find_analysis_bucket_files( - filter={"analysis_id": analysis_code_bucket.analysis_id} + filter={"analysisId": analysis_code_bucket.analysis_id} ).pop() yield analysis_bucket_file @@ -224,11 +224,11 @@ def analysis_log(core_client, configured_analysis): core_client.send_analysis_command(configured_analysis, "buildStart") def _check_analysis_logs_present(): - assert len(core_client.find_analysis_logs(filter={"analysis_id": configured_analysis.id})) > 0 + assert len(core_client.find_analysis_logs(filter={"analysisId": configured_analysis.id})) > 0 assert_eventually(_check_analysis_logs_present) - return core_client.find_analysis_logs(filter={"analysis_id": configured_analysis.id})[0] + return core_client.find_analysis_logs(filter={"analysisId": configured_analysis.id})[0] @pytest.fixture() @@ -394,7 +394,7 @@ def test_get_project_nodes(core_client, project_node, project_node_includables): def test_find_project_nodes(core_client, project_node, project_node_includables): # Use "project_id" instead of "id" because filtering for ids does not work. - project_nodes_find = core_client.find_project_nodes(filter={"project_id": project_node.project_id}) + project_nodes_find = core_client.find_project_nodes(filter={"projectId": project_node.project_id}) assert [project_node.id] == [pn.id for pn in project_nodes_find] assert all( @@ -553,7 +553,7 @@ def test_get_analysis_nodes(core_client, analysis_node, analysis_node_includable def test_find_analysis_nodes(core_client, analysis_node, analysis_node_includables): # Use "analysis_id" instead of "id" because filtering for ids does not work. - analysis_nodes_find = core_client.find_analysis_nodes(filter={"analysis_id": analysis_node.analysis_id}) + analysis_nodes_find = core_client.find_analysis_nodes(filter={"analysisId": analysis_node.analysis_id}) assert [analysis_node.id] == [an.id for an in analysis_nodes_find] assert all( @@ -584,14 +584,14 @@ def test_analysis_node_logs(core_client, analysis_node): def _check_analysis_node_logs_present(): found_logs = core_client.find_analysis_node_logs( - filter={"analysis_id": analysis_node.analysis_id, "node_id": analysis_node.node_id} + filter={"analysisId": analysis_node.analysis_id, "nodeId": analysis_node.node_id} ) assert len(found_logs) == 1 assert_eventually(_check_analysis_node_logs_present) new_log = core_client.find_analysis_node_logs( - filter={"analysis_id": analysis_node.analysis_id, "node_id": analysis_node.node_id} + filter={"analysisId": analysis_node.analysis_id, "nodeId": analysis_node.node_id} )[0] assert log == new_log @@ -601,7 +601,7 @@ def _check_analysis_node_logs_present(): assert ( len( core_client.find_analysis_node_logs( - filter={"analysis_id": analysis_node.analysis_id, "node_id": analysis_node.node_id} + filter={"analysisId": analysis_node.analysis_id, "nodeId": analysis_node.node_id} ) ) == 0 @@ -631,7 +631,7 @@ def test_get_analysis_buckets(core_client, analysis_code_bucket, analysis_bucket def test_find_analysis_buckets(core_client, analysis_code_bucket, analysis_bucket_includables): # Use "analysis_id" instead of "id" because filtering for ids does not work. - analysis_buckets_find = core_client.find_analysis_buckets(filter={"analysis_id": analysis_code_bucket.analysis_id}) + analysis_buckets_find = core_client.find_analysis_buckets(filter={"analysisId": analysis_code_bucket.analysis_id}) assert analysis_code_bucket.id in [bucket.id for bucket in analysis_buckets_find] assert all( @@ -669,7 +669,7 @@ def test_get_analysis_bucket_files(core_client, analysis_bucket_file, analysis_b def test_find_analysis_bucket_files(core_client, analysis_bucket_file, analysis_bucket_file_includables): # Use "analysis_id" instead of "id" because filtering for ids does not work. analysis_bucket_files_find = core_client.find_analysis_bucket_files( - filter={"analysis_id": analysis_bucket_file.analysis_id} + filter={"analysisId": analysis_bucket_file.analysis_id} ) assert [analysis_bucket_file.id] == [abf.id for abf in analysis_bucket_files_find] @@ -726,7 +726,7 @@ def test_registry_setup(core_client, registry): core_client.send_registry_command(registry.id, command="setup") def _check_setup(): - registry_projects = core_client.find_registry_projects(filter={"registry_id": registry.id}) + registry_projects = core_client.find_registry_projects(filter={"registryId": registry.id}) assert len(registry_projects) == 3 assert {"incoming", "outgoing", "masterImages"} == set(rp.type for rp in registry_projects) @@ -780,6 +780,6 @@ def test_update_registry_project(core_client, registry_project): @pytest.mark.xfail(reason="Bug in Hub, see https://github.com/PrivateAIM/hub/issues/1181.") def test_delete_analysis_logs(core_client, analysis_log): - core_client.delete_analysis_logs(analysis_id=analysis_log.labels["analysis_id"]) + core_client.delete_analysis_logs(analysis_id=analysis_log.labels["analysisId"]) - assert len(core_client.find_analysis_logs(filter={"analysis_id": analysis_log.labels["analysis_id"]})) == 0 + assert len(core_client.find_analysis_logs(filter={"analysisId": analysis_log.labels["analysisId"]})) == 0