From 05342ff1f6f01a8b003d9982140e2d01e0009a22 Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:58:09 +0330 Subject: [PATCH 01/22] Update keycloak opex realm configuration (#696) --- .../keycloak-setup/realms/opex-realm.json | 1071 +++++++++++++---- 1 file changed, 839 insertions(+), 232 deletions(-) diff --git a/auth-gateway/keycloak-setup/realms/opex-realm.json b/auth-gateway/keycloak-setup/realms/opex-realm.json index c26448246..9644f98e0 100644 --- a/auth-gateway/keycloak-setup/realms/opex-realm.json +++ b/auth-gateway/keycloak-setup/realms/opex-realm.json @@ -1,11 +1,13 @@ { "id": "opex", "realm": "opex", + "displayName": "", + "displayNameHtml": "", "notBefore": 0, "defaultSignatureAlgorithm": "RS256", "revokeRefreshToken": false, "refreshTokenMaxReuse": 0, - "accessTokenLifespan": 1800, + "accessTokenLifespan": 900, "accessTokenLifespanForImplicitFlow": 900, "ssoSessionIdleTimeout": 1209600, "ssoSessionMaxLifespan": 1209600, @@ -18,7 +20,7 @@ "clientSessionMaxLifespan": 0, "clientOfflineSessionIdleTimeout": 0, "clientOfflineSessionMaxLifespan": 0, - "accessCodeLifespan": 60, + "accessCodeLifespan": 120, "accessCodeLifespanUserAction": 300, "accessCodeLifespanLogin": 1800, "actionTokenGeneratedByAdminLifespan": 43200, @@ -59,7 +61,11 @@ }, "clientRole": false, "containerId": "opex", - "attributes": {} + "attributes": { + "permissions": [ + "all" + ] + } }, { "id": "77c38736-27cc-4787-b83c-6b30872df6f7", @@ -106,6 +112,9 @@ "description": "${role_default-roles}", "composite": true, "composites": { + "realm": [ + "user-1" + ], "client": { "account": [ "manage-account", @@ -117,6 +126,15 @@ "containerId": "opex", "attributes": {} }, + { + "id": "43ffe3ec-76a6-4e62-bfc9-213916087101", + "name": "monitoring", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "opex", + "attributes": {} + }, { "id": "3b6109f5-6e5a-4578-83c3-791ec3e2bf9e", "name": "offline_access", @@ -126,6 +144,15 @@ "containerId": "opex", "attributes": {} }, + { + "id": "08a06bd3-9427-4fa2-8b6e-46009e68956a", + "name": "rate_bot", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "opex", + "attributes": {} + }, { "id": "0dd6a8c7-d669-4941-9ea1-521980e9c53f", "name": "uma_authorization", @@ -149,7 +176,7 @@ "containerId": "opex", "attributes": { "permissions": [ - "withdraw:write,voucher:submit" + "withdraw:write,voucher:submit,bank_account:write" ] } }, @@ -388,7 +415,7 @@ "ios-app": [], "opex-api-key": [ { - "id": "95d01e3b-1442-415c-9f86-4d86187558ca", + "id": "e0c7c621-6d27-4a63-9a4e-c0e6ccf22d1a", "name": "uma_protection", "composite": false, "clientRole": true, @@ -397,6 +424,17 @@ } ], "security-admin-console": [], + "pre-auth-client": [ + { + "id": "b01b531f-e02d-4a4c-806b-2503bc3b0f21", + "name": "uma_protection", + "description": "", + "composite": false, + "clientRole": true, + "containerId": "bfc7873d-7250-41aa-bf8d-36966b7614bb", + "attributes": {} + } + ], "admin-cli": [], "account-console": [], "android-app": [], @@ -413,7 +451,7 @@ ], "opex-admin": [ { - "id": "cd0dca5f-aa89-4a97-8cb7-6525f919c3b6", + "id": "8795818a-526e-4235-b55c-afa71d2f2c09", "name": "uma_protection", "composite": false, "clientRole": true, @@ -520,6 +558,17 @@ "attributes": {}, "realmRoles": [], "clientRoles": {} + }, + { + "id": "0d3170da-25f4-4cd9-9f38-62937665e65d", + "name": "offline_access", + "path": "/offline_access", + "subGroups": [], + "attributes": {}, + "realmRoles": [ + "offline_access" + ], + "clientRoles": {} } ], "defaultRole": { @@ -533,7 +582,7 @@ "requiredCredentials": [ "password" ], - "passwordPolicy": "length(8)", + "passwordPolicy": "length(8) and regexPattern(^(?=.*[a-z])(?=.*[A-Z])(?=.*\\d)(?=.*[^A-Za-z0-9]).{8,}$) and notContainsUsername(undefined)", "otpPolicyType": "totp", "otpPolicyAlgorithm": "HmacSHA1", "otpPolicyInitialCounter": 0, @@ -576,7 +625,7 @@ "users": [ { "id": "cb6af759-5e4f-42b3-86ea-b3754ce4d422", - "username": "service-account-account-console", + "username": "43033518", "emailVerified": false, "createdTimestamp": 1624136397065, "enabled": true, @@ -602,7 +651,7 @@ }, { "id": "2cda7f75-c5d9-4b64-b90e-58b381689a9d", - "username": "service-account-opex-admin", + "username": "10101003", "emailVerified": false, "createdTimestamp": 1643624421752, "enabled": true, @@ -631,10 +680,10 @@ "groups": [] }, { - "id": "4bb55f60-faa0-464a-b0a3-04a18f421dff", - "username": "service-account-opex-api-key", + "id": "7915a5da-5ed2-444e-b63c-960e17a47fcd", + "username": "59153225", "emailVerified": false, - "createdTimestamp": 1749460715817, + "createdTimestamp": 1749472157240, "enabled": true, "totp": false, "serviceAccountClientId": "opex-api-key", @@ -644,7 +693,28 @@ "default-roles-opex" ], "clientRoles": { - "opex-api-key": [ + "realm-management": [ + "impersonation" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "id": "5c2f4d42-f92f-4e30-a137-806d27a1027b", + "username": "81120214", + "emailVerified": false, + "createdTimestamp": 1765968142838, + "enabled": true, + "totp": false, + "serviceAccountClientId": "pre-auth-client", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-opex" + ], + "clientRoles": { + "pre-auth-client": [ "uma_protection" ] }, @@ -653,7 +723,7 @@ }, { "id": "854a5e2b-1e45-4ff7-bd1c-d9764d41a5bd", - "username": "service-account-realm-management", + "username": "59179637", "emailVerified": false, "createdTimestamp": 1634844207750, "enabled": true, @@ -931,10 +1001,7 @@ "id": "13d76feb-d762-4409-bb84-7a75bc395a61", "clientId": "admin-cli", "name": "${client_admin-cli}", - "description": "", "rootUrl": "", - "adminUrl": "", - "baseUrl": "", "surrogateAuthRequired": false, "enabled": true, "alwaysDisplayInConsole": false, @@ -954,36 +1021,24 @@ "frontchannelLogout": false, "protocol": "openid-connect", "attributes": { - "request.object.signature.alg": "any", + "saml.assertion.signature": "false", "saml.multivalued.roles": "false", "saml.force.post.binding": "false", + "saml.encrypt": "false", "post.logout.redirect.uris": "+", - "oauth2.device.authorization.grant.enabled": "false", + "saml.server.signature": "false", "backchannel.logout.revoke.offline.tokens": "false", "saml.server.signature.keyinfo.ext": "false", - "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", "realm_client": "false", - "oidc.ciba.grant.enabled": "false", + "client.use.lightweight.access.token.enabled": "true", "backchannel.logout.session.required": "false", "client_credentials.use_refresh_token": "false", - "saml.client.signature": "false", - "require.pushed.authorization.requests": "false", - "request.object.encryption.enc": "any", - "saml.assertion.signature": "false", - "client.secret.creation.time": "1749563219", - "request.object.encryption.alg": "any", - "client.introspection.response.allow.jwt.claim.enabled": "false", - "saml.encrypt": "false", - "saml.server.signature": "false", - "exclude.session.state.from.auth.response": "false", - "client.use.lightweight.access.token.enabled": "true", - "request.object.required": "not required", "saml_force_name_id_format": "false", + "saml.client.signature": "false", "tls.client.certificate.bound.access.tokens": "false", - "acr.loa.map": "{}", "saml.authnstatement": "false", "display.on.consent.screen": "false", - "token.response.type.bearer.lower-case": "false", "saml.onetimeuse.condition": "false" }, "authenticationFlowBindingOverrides": {}, @@ -1111,6 +1166,7 @@ "access.token.lifespan": "600", "request.object.signature.alg": "any", "frontchannel.logout.session.required": "true", + "post.logout.redirect.uris": "+", "oauth2.device.authorization.grant.enabled": "false", "use.jwks.url": "false", "backchannel.logout.revoke.offline.tokens": "false", @@ -1123,7 +1179,7 @@ "require.pushed.authorization.requests": "false", "request.object.encryption.enc": "any", "client.session.max.lifespan": "604800", - "client.secret.creation.time": "1747492073", + "client.secret.creation.time": "1749304241", "request.object.encryption.alg": "any", "client.introspection.response.allow.jwt.claim.enabled": "false", "exclude.session.state.from.auth.response": "false", @@ -1134,12 +1190,16 @@ "display.on.consent.screen": "false", "token.response.type.bearer.lower-case": "false" }, - "authenticationFlowBindingOverrides": {}, + "authenticationFlowBindingOverrides": { + "direct_grant": "12db0ccb-9df4-4173-95fc-94650a4167e4" + }, "fullScopeAllowed": true, "nodeReRegistrationTimeout": -1, "defaultClientScopes": [ "trust", + "Audience_Resolver", "basic", + "android-app-force-audience", "role_permission_attribute" ], "optionalClientScopes": [] @@ -1241,6 +1301,7 @@ "access.token.lifespan": "600", "request.object.signature.alg": "any", "frontchannel.logout.session.required": "true", + "post.logout.redirect.uris": "+", "oauth2.device.authorization.grant.enabled": "false", "use.jwks.url": "false", "backchannel.logout.revoke.offline.tokens": "false", @@ -1253,7 +1314,7 @@ "require.pushed.authorization.requests": "false", "request.object.encryption.enc": "any", "client.session.max.lifespan": "604800", - "client.secret.creation.time": "1747492982", + "client.secret.creation.time": "1749304288", "request.object.encryption.alg": "any", "client.introspection.response.allow.jwt.claim.enabled": "false", "exclude.session.state.from.auth.response": "false", @@ -1264,11 +1325,15 @@ "display.on.consent.screen": "false", "token.response.type.bearer.lower-case": "false" }, - "authenticationFlowBindingOverrides": {}, + "authenticationFlowBindingOverrides": { + "direct_grant": "12db0ccb-9df4-4173-95fc-94650a4167e4" + }, "fullScopeAllowed": true, "nodeReRegistrationTimeout": -1, "defaultClientScopes": [ "trust", + "Audience_Resolver", + "ios-app-force-audience", "basic", "role_permission_attribute" ], @@ -1303,27 +1368,37 @@ "frontchannelLogout": false, "protocol": "openid-connect", "attributes": { - "saml.assertion.signature": "false", "access.token.lifespan": "86400", - "client.secret.creation.time": "1745840695", + "request.object.signature.alg": "any", "saml.multivalued.roles": "false", "saml.force.post.binding": "false", - "saml.encrypt": "false", "post.logout.redirect.uris": "+", "oauth2.device.authorization.grant.enabled": "false", "backchannel.logout.revoke.offline.tokens": "false", - "saml.server.signature": "false", "saml.server.signature.keyinfo.ext": "false", - "exclude.session.state.from.auth.response": "false", + "use.refresh.tokens": "true", "realm_client": "false", "oidc.ciba.grant.enabled": "false", "backchannel.logout.session.required": "false", "client_credentials.use_refresh_token": "false", - "saml_force_name_id_format": "false", "saml.client.signature": "false", + "require.pushed.authorization.requests": "false", + "request.object.encryption.enc": "any", + "saml.assertion.signature": "false", + "client.secret.creation.time": "1770816326", + "request.object.encryption.alg": "any", + "client.introspection.response.allow.jwt.claim.enabled": "false", + "saml.encrypt": "false", + "saml.server.signature": "false", + "exclude.session.state.from.auth.response": "false", + "client.use.lightweight.access.token.enabled": "false", + "request.object.required": "not required", + "saml_force_name_id_format": "false", "tls.client.certificate.bound.access.tokens": "false", + "acr.loa.map": "{}", "saml.authnstatement": "false", "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false", "saml.onetimeuse.condition": "false" }, "authenticationFlowBindingOverrides": {}, @@ -1413,16 +1488,6 @@ } ], "policies": [ - { - "name": "Default Policy", - "description": "A policy that grants access only for users within this realm", - "type": "js", - "logic": "POSITIVE", - "decisionStrategy": "AFFIRMATIVE", - "config": { - "code": "// by default, grants any permission associated with this policy\n$evaluation.grant();\n" - } - }, { "name": "Default Permission", "description": "A permission that applies to the default resource type", @@ -1430,8 +1495,7 @@ "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "defaultResourceType": "urn:opex-admin:resources:default", - "applyPolicies": "[\"Default Policy\"]" + "defaultResourceType": "urn:opex-admin:resources:default" } } ], @@ -1463,86 +1527,132 @@ "implicitFlowEnabled": false, "directAccessGrantsEnabled": true, "serviceAccountsEnabled": true, - "authorizationServicesEnabled": true, "publicClient": false, "frontchannelLogout": false, "protocol": "openid-connect", "attributes": { - "saml.assertion.signature": "false", "access.token.lifespan": "43200", - "client.secret.creation.time": "1749460384", + "request.object.signature.alg": "any", "saml.multivalued.roles": "false", "saml.force.post.binding": "false", - "saml.encrypt": "false", "post.logout.redirect.uris": "+", "oauth2.device.authorization.grant.enabled": "false", + "use.jwks.url": "false", "backchannel.logout.revoke.offline.tokens": "false", - "saml.server.signature": "false", "saml.server.signature.keyinfo.ext": "false", - "exclude.session.state.from.auth.response": "false", + "use.refresh.tokens": "true", "realm_client": "false", "oidc.ciba.grant.enabled": "false", "backchannel.logout.session.required": "true", "client_credentials.use_refresh_token": "false", - "saml_force_name_id_format": "false", + "client.offline.session.idle.timeout": "2592000", "saml.client.signature": "false", + "require.pushed.authorization.requests": "false", + "request.object.encryption.enc": "any", + "client.session.max.lifespan": "2592000", + "client.session.idle.timeout": "2592000", + "saml.assertion.signature": "false", + "client.secret.creation.time": "1770816360", + "request.object.encryption.alg": "any", + "client.introspection.response.allow.jwt.claim.enabled": "false", + "saml.encrypt": "false", + "saml.server.signature": "false", + "exclude.session.state.from.auth.response": "false", + "client.use.lightweight.access.token.enabled": "false", + "request.object.required": "not required", + "saml_force_name_id_format": "false", "tls.client.certificate.bound.access.tokens": "false", + "acr.loa.map": "{}", "saml.authnstatement": "false", "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false", "saml.onetimeuse.condition": "false" }, "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, + "fullScopeAllowed": true, "nodeReRegistrationTimeout": -1, "defaultClientScopes": [ + "service_account", "trust", - "Forced_Roles", + "Audience_Resolver", + "web-origins", + "acr", + "profile", + "api-key-force-audience", "basic", + "email", "role_permission_attribute" ], "optionalClientScopes": [ - "offline_access" + "address", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "bfc7873d-7250-41aa-bf8d-36966b7614bb", + "clientId": "pre-auth-client", + "name": "pre-auth-client", + "description": "", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "/*" ], - "authorizationSettings": { - "allowRemoteResourceManagement": true, - "policyEnforcementMode": "ENFORCING", - "resources": [ - { - "name": "Default Resource", - "type": "urn:opex-api-key:resources:default", - "ownerManagedAccess": false, - "attributes": {}, - "uris": [ - "/*" - ] - } - ], - "policies": [ - { - "name": "Default Policy", - "description": "A policy that grants access only for users within this realm", - "type": "js", - "logic": "POSITIVE", - "decisionStrategy": "AFFIRMATIVE", - "config": { - "code": "// by default, grants any permission associated with this policy\n$evaluation.grant();\n" - } - }, - { - "name": "Default Permission", - "description": "A permission that applies to the default resource type", - "type": "resource", - "logic": "POSITIVE", - "decisionStrategy": "UNANIMOUS", - "config": { - "defaultResourceType": "urn:opex-api-key:resources:default", - "applyPolicies": "[\"Default Policy\"]" - } - } - ], - "scopes": [], - "decisionStrategy": "UNANIMOUS" - } + "webOrigins": [ + "/*" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "access.token.lifespan": "300", + "request.object.signature.alg": "any", + "client.secret.creation.time": "1770816293", + "request.object.encryption.alg": "any", + "client.introspection.response.allow.jwt.claim.enabled": "false", + "frontchannel.logout.session.required": "true", + "post.logout.redirect.uris": "+", + "oauth2.device.authorization.grant.enabled": "false", + "use.jwks.url": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "use.refresh.tokens": "true", + "realm_client": "false", + "oidc.ciba.grant.enabled": "false", + "client.use.lightweight.access.token.enabled": "false", + "backchannel.logout.session.required": "true", + "request.object.required": "not required", + "client_credentials.use_refresh_token": "false", + "tls.client.certificate.bound.access.tokens": "false", + "require.pushed.authorization.requests": "false", + "acr.loa.map": "{}", + "display.on.consent.screen": "false", + "request.object.encryption.enc": "any", + "token.response.type.bearer.lower-case": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "service_account", + "Audience_Resolver", + "pre-auth-client-force-audience", + "basic" + ], + "optionalClientScopes": [] }, { "id": "6a4bfbd0-576d-4778-af56-56f876647355", @@ -1637,7 +1747,7 @@ "policyEnforcementMode": "ENFORCING", "resources": [ { - "name": "client.resource.13d76feb-d762-4409-bb84-7a75bc395a61", + "name": "client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976", "type": "Client", "ownerManagedAccess": false, "attributes": {}, @@ -1659,15 +1769,15 @@ "name": "manage" }, { - "name": "token-exchange" + "name": "map-roles-composite" }, { - "name": "map-roles-composite" + "name": "token-exchange" } ] }, { - "name": "client.resource.ceabb7ca-b063-4755-90fb-8de2cc7e5e00", + "name": "client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da", "type": "Client", "ownerManagedAccess": false, "attributes": {}, @@ -1689,41 +1799,57 @@ "name": "manage" }, { - "name": "token-exchange" + "name": "map-roles-composite" }, { - "name": "map-roles-composite" + "name": "token-exchange" } ] }, { - "name": "Users", + "name": "idp.resource.6456448e-2415-49ad-bf95-1b5176557862", + "type": "IdentityProvider", + "ownerManagedAccess": false, + "attributes": {}, + "uris": [], + "scopes": [ + { + "name": "token-exchange" + } + ] + }, + { + "name": "client.resource.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "Client", "ownerManagedAccess": false, "attributes": {}, "uris": [], "scopes": [ { - "name": "user-impersonated" - }, - { - "name": "manage-group-membership" - }, - { "name": "view" }, { - "name": "impersonate" + "name": "map-roles-client-scope" + }, + { + "name": "configure" }, { "name": "map-roles" }, { "name": "manage" + }, + { + "name": "map-roles-composite" + }, + { + "name": "token-exchange" } ] }, { - "name": "client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "Client", "ownerManagedAccess": false, "attributes": {}, @@ -1744,16 +1870,46 @@ { "name": "manage" }, + { + "name": "map-roles-composite" + }, { "name": "token-exchange" + } + ] + }, + { + "name": "client.resource.13d76feb-d762-4409-bb84-7a75bc395a61", + "type": "Client", + "ownerManagedAccess": false, + "attributes": {}, + "uris": [], + "scopes": [ + { + "name": "view" + }, + { + "name": "map-roles-client-scope" + }, + { + "name": "configure" + }, + { + "name": "map-roles" + }, + { + "name": "manage" }, { "name": "map-roles-composite" + }, + { + "name": "token-exchange" } ] }, { - "name": "client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "client.resource.ceabb7ca-b063-4755-90fb-8de2cc7e5e00", "type": "Client", "ownerManagedAccess": false, "attributes": {}, @@ -1774,21 +1930,65 @@ { "name": "manage" }, + { + "name": "map-roles-composite" + }, { "name": "token-exchange" + } + ] + }, + { + "name": "Users", + "ownerManagedAccess": false, + "attributes": {}, + "uris": [], + "scopes": [ + { + "name": "user-impersonated" }, { - "name": "map-roles-composite" + "name": "view" + }, + { + "name": "manage-group-membership" + }, + { + "name": "impersonate" + }, + { + "name": "map-roles" + }, + { + "name": "manage" } ] }, { - "name": "idp.resource.6456448e-2415-49ad-bf95-1b5176557862", - "type": "IdentityProvider", + "name": "client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c", + "type": "Client", "ownerManagedAccess": false, "attributes": {}, "uris": [], "scopes": [ + { + "name": "view" + }, + { + "name": "map-roles-client-scope" + }, + { + "name": "configure" + }, + { + "name": "map-roles" + }, + { + "name": "manage" + }, + { + "name": "map-roles-composite" + }, { "name": "token-exchange" } @@ -1814,6 +2014,26 @@ "clients": "[\"opex-admin\"]" } }, + { + "name": "allow-pre-auth-client", + "description": "", + "type": "client", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "clients": "[\"pre-auth-client\"]" + } + }, + { + "name": "pre-auth-policy", + "description": "", + "type": "client", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "clients": "[\"pre-auth-client\"]" + } + }, { "name": "manage.permission.client.13d76feb-d762-4409-bb84-7a75bc395a61", "type": "scope", @@ -1885,25 +2105,25 @@ } }, { - "name": "token-exchange.permission.idp.6456448e-2415-49ad-bf95-1b5176557862", - "description": "", + "name": "manage.permission.client.ceabb7ca-b063-4755-90fb-8de2cc7e5e00", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"idp.resource.6456448e-2415-49ad-bf95-1b5176557862\"]", - "scopes": "[\"token-exchange\"]", - "applyPolicies": "[\"opex-api-exchange\"]" + "resources": "[\"client.resource.ceabb7ca-b063-4755-90fb-8de2cc7e5e00\"]", + "scopes": "[\"manage\"]" } }, { - "name": "manage.permission.client.ceabb7ca-b063-4755-90fb-8de2cc7e5e00", + "name": "token-exchange.permission.idp.6456448e-2415-49ad-bf95-1b5176557862", + "description": "", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.ceabb7ca-b063-4755-90fb-8de2cc7e5e00\"]", - "scopes": "[\"manage\"]" + "resources": "[\"idp.resource.6456448e-2415-49ad-bf95-1b5176557862\"]", + "scopes": "[\"token-exchange\"]", + "applyPolicies": "[\"opex-api-exchange\"]" } }, { @@ -2002,170 +2222,386 @@ "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"Users\"]", - "scopes": "[\"manage-group-membership\"]" + "resources": "[\"Users\"]", + "scopes": "[\"manage-group-membership\"]" + } + }, + { + "name": "admin-impersonating.permission.users", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"Users\"]", + "scopes": "[\"impersonate\"]", + "applyPolicies": "[\"account-console-client-impersonate\"]" + } + }, + { + "name": "user-impersonated.permission.users", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"Users\"]", + "scopes": "[\"user-impersonated\"]" + } + }, + { + "name": "view.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"view\"]" + } + }, + { + "name": "manage.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"manage\"]" + } + }, + { + "name": "configure.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"configure\"]" + } + }, + { + "name": "map-roles.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"map-roles\"]" + } + }, + { + "name": "map-roles-client-scope.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"map-roles-client-scope\"]" + } + }, + { + "name": "map-roles-composite.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"map-roles-composite\"]" + } + }, + { + "name": "token-exchange.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "scopes": "[\"token-exchange\"]" + } + }, + { + "name": "manage.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"manage\"]" + } + }, + { + "name": "configure.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"configure\"]" + } + }, + { + "name": "view.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"view\"]" + } + }, + { + "name": "map-roles.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"map-roles\"]" + } + }, + { + "name": "map-roles-client-scope.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"map-roles-client-scope\"]" + } + }, + { + "name": "map-roles-composite.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"map-roles-composite\"]" + } + }, + { + "name": "token-exchange.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "scopes": "[\"token-exchange\"]", + "applyPolicies": "[\"opex-api-exchange\"]" + } + }, + { + "name": "manage.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"manage\"]" + } + }, + { + "name": "configure.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"configure\"]" + } + }, + { + "name": "view.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"view\"]" + } + }, + { + "name": "map-roles.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"map-roles\"]" + } + }, + { + "name": "map-roles-client-scope.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "type": "scope", + "logic": "POSITIVE", + "decisionStrategy": "UNANIMOUS", + "config": { + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"map-roles-client-scope\"]" } }, { - "name": "admin-impersonating.permission.users", + "name": "map-roles-composite.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"Users\"]", - "scopes": "[\"impersonate\"]", - "applyPolicies": "[\"account-console-client-impersonate\"]" + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"map-roles-composite\"]" } }, { - "name": "user-impersonated.permission.users", + "name": "manage.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"Users\"]", - "scopes": "[\"user-impersonated\"]" + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", + "scopes": "[\"manage\"]" } }, { - "name": "view.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "token-exchange.permission.client.fd116873-8b00-4851-a88d-1a72575783ba", + "description": "", "type": "scope", "logic": "POSITIVE", - "decisionStrategy": "UNANIMOUS", + "decisionStrategy": "AFFIRMATIVE", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", - "scopes": "[\"view\"]" + "resources": "[\"client.resource.fd116873-8b00-4851-a88d-1a72575783ba\"]", + "scopes": "[\"token-exchange\"]", + "applyPolicies": "[\"allow-pre-auth-client\"]" } }, { - "name": "manage.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "configure.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", - "scopes": "[\"manage\"]" + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", + "scopes": "[\"configure\"]" } }, { - "name": "configure.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "view.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", - "scopes": "[\"configure\"]" + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", + "scopes": "[\"view\"]" } }, { - "name": "map-roles.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "map-roles.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", "scopes": "[\"map-roles\"]" } }, { - "name": "map-roles-client-scope.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "map-roles-client-scope.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", "scopes": "[\"map-roles-client-scope\"]" } }, { - "name": "map-roles-composite.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "map-roles-composite.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", "scopes": "[\"map-roles-composite\"]" } }, { - "name": "token-exchange.permission.client.fb5f91c4-42fa-4769-b45d-febef22b4976", + "name": "manage.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.fb5f91c4-42fa-4769-b45d-febef22b4976\"]", - "scopes": "[\"token-exchange\"]" + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", + "scopes": "[\"manage\"]" } }, { - "name": "manage.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "token-exchange.permission.client.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6", + "description": "", "type": "scope", "logic": "POSITIVE", - "decisionStrategy": "UNANIMOUS", + "decisionStrategy": "AFFIRMATIVE", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", - "scopes": "[\"manage\"]" + "resources": "[\"client.resource.01ca7824-0953-4d5f-acf5-99fbf0ef2eb6\"]", + "scopes": "[\"token-exchange\"]", + "applyPolicies": "[\"allow-pre-auth-client\"]" } }, { - "name": "configure.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "configure.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"configure\"]" } }, { - "name": "view.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "view.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"view\"]" } }, { - "name": "map-roles.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "map-roles.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"map-roles\"]" } }, { - "name": "map-roles-client-scope.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "map-roles-client-scope.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"map-roles-client-scope\"]" } }, { - "name": "map-roles-composite.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "map-roles-composite.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", "type": "scope", "logic": "POSITIVE", "decisionStrategy": "UNANIMOUS", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"map-roles-composite\"]" } }, { - "name": "token-exchange.permission.client.d2f0f1b6-46b7-4678-842a-8c67524ea2da", + "name": "token-exchange.permission.client.30c68e47-1e37-4ce4-a323-efec3dcec65c", + "description": "", "type": "scope", "logic": "POSITIVE", - "decisionStrategy": "UNANIMOUS", + "decisionStrategy": "AFFIRMATIVE", "config": { - "resources": "[\"client.resource.d2f0f1b6-46b7-4678-842a-8c67524ea2da\"]", + "resources": "[\"client.resource.30c68e47-1e37-4ce4-a323-efec3dcec65c\"]", "scopes": "[\"token-exchange\"]", - "applyPolicies": "[\"opex-api-exchange\"]" + "applyPolicies": "[\"allow-pre-auth-client\"]" } } ], @@ -2326,6 +2762,7 @@ "access.token.lifespan": "3600", "request.object.signature.alg": "any", "frontchannel.logout.session.required": "true", + "post.logout.redirect.uris": "+", "oauth2.device.authorization.grant.enabled": "false", "use.jwks.url": "false", "backchannel.logout.revoke.offline.tokens": "false", @@ -2341,7 +2778,6 @@ "client.secret.creation.time": "1747492073", "request.object.encryption.alg": "any", "client.introspection.response.allow.jwt.claim.enabled": "false", - "standard.token.exchange.enabled": "false", "exclude.session.state.from.auth.response": "false", "client.use.lightweight.access.token.enabled": "false", "request.object.required": "not required", @@ -2350,19 +2786,51 @@ "display.on.consent.screen": "false", "token.response.type.bearer.lower-case": "false" }, - "authenticationFlowBindingOverrides": {}, + "authenticationFlowBindingOverrides": { + "direct_grant": "12db0ccb-9df4-4173-95fc-94650a4167e4" + }, "fullScopeAllowed": true, "nodeReRegistrationTimeout": -1, "defaultClientScopes": [ "trust", - "audience-opex-api-key", + "Audience_Resolver", "basic", + "web-app-force-audience", "role_permission_attribute" ], "optionalClientScopes": [] } ], "clientScopes": [ + { + "id": "6bed3324-5c8f-4f0b-b801-69f60537e8f7", + "name": "web-app-force-audience", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "53b4b835-110b-45c8-a168-2cfd28111c4f", + "name": "audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "web-app", + "id.token.claim": "false", + "lightweight.claim": "false", + "introspection.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "false" + } + } + ] + }, { "id": "0cbd4466-de57-4fc9-81a7-f34f3cd2262a", "name": "service_account", @@ -2423,6 +2891,35 @@ } ] }, + { + "id": "dba29456-b27e-4e02-bdb3-e195c15dd924", + "name": "pre-auth-client-force-audience", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "188d8065-b18d-4f38-ae6f-79a2838d3ecc", + "name": "audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "pre-auth-client", + "id.token.claim": "false", + "lightweight.claim": "false", + "introspection.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "false" + } + } + ] + }, { "id": "05ec9110-1046-4784-ae97-d8bf86bcfc62", "name": "acr", @@ -2669,37 +3166,49 @@ ] }, { - "id": "d1debc5e-632b-4c2e-863b-2f5b2b1572d5", - "name": "user-roles", + "id": "0e147d46-11d0-4e9b-b291-b4063d038b09", + "name": "api-key-force-audience", "description": "", "protocol": "openid-connect", "attributes": { - "include.in.token.scope": "true", + "include.in.token.scope": "false", "display.on.consent.screen": "true", "gui.order": "", "consent.screen.text": "" }, "protocolMappers": [ { - "id": "d8aa4645-5f5b-41e7-b7e6-b12739ca0cca", - "name": "realm roles", + "id": "1533abb2-466c-4ef2-a542-7717412d5551", + "name": "audience", "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-realm-role-mapper", + "protocolMapper": "oidc-audience-mapper", "consentRequired": false, "config": { - "user.attribute": "foo", + "included.client.audience": "opex-api-key", + "id.token.claim": "false", + "lightweight.claim": "false", "introspection.token.claim": "true", "access.token.claim": "true", - "claim.name": "realm_access.roles", - "jsonType.label": "String", - "multivalued": "true" + "userinfo.token.claim": "false" } } ] }, { - "id": "dbb7860d-fba0-4c05-8b99-f2f9cdd3ffb4", - "name": "audience-opex-api-key", + "id": "26449071-f693-40a0-b814-60a62e7384a5", + "name": "audience-opex-web-app", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + } + }, + { + "id": "c3a0bc7d-d53a-46e7-bf10-dce083cf1a06", + "name": "android-app-force-audience", "description": "", "protocol": "openid-connect", "attributes": { @@ -2710,15 +3219,41 @@ }, "protocolMappers": [ { - "id": "627885b9-1bb8-465a-9125-e9603ac1dcd5", - "name": "audience-opex-api-key", + "id": "5abd6a8f-902d-40e9-83dc-20f93bcb636e", + "name": "audience", "protocol": "openid-connect", "protocolMapper": "oidc-audience-mapper", "consentRequired": false, "config": { - "included.client.audience": "opex-api-key", + "included.client.audience": "android-app", "id.token.claim": "false", "lightweight.claim": "false", + "introspection.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "false" + } + } + ] + }, + { + "id": "cd271546-135d-4006-810f-b1d212b36bd6", + "name": "Audience_Resolver", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "c8c4decd-67fb-4569-a320-0b67fa9aac59", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": { "access.token.claim": "true", "introspection.token.claim": "true" } @@ -2782,17 +3317,6 @@ "claim.name": "auth_time", "jsonType.label": "long" } - }, - { - "id": "f4b67796-04fa-49b5-846f-a5ab81587c55", - "name": "audience resolve", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-resolve-mapper", - "consentRequired": false, - "config": { - "introspection.token.claim": "true", - "access.token.claim": "true" - } } ] }, @@ -3061,6 +3585,35 @@ } ] }, + { + "id": "1fdced46-2804-4d1c-8b51-c00617a55dae", + "name": "audience-opex-api-key", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "481a6bf3-ae13-4ce6-97c4-68e7119e56aa", + "name": "audience-opex-api-key", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "opex-api-key", + "id.token.claim": "false", + "lightweight.claim": "false", + "introspection.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "false" + } + } + ] + }, { "id": "b3526ac1-10e2-4344-8621-9c5a0853e97a", "name": "email", @@ -3131,8 +3684,8 @@ ] }, { - "id": "60575b8d-b5ab-47c3-a2fa-b14e77d4f392", - "name": "Forced_Roles", + "id": "40951a99-ac8b-476d-bf96-3361f69afb2d", + "name": "ios-app-force-audience", "description": "", "protocol": "openid-connect", "attributes": { @@ -3143,15 +3696,18 @@ }, "protocolMappers": [ { - "id": "a2ffe329-0ed7-4861-a6aa-e72a25b98770", - "name": "roles", + "id": "6b7903e9-a540-431e-b3aa-ba1ebf2bbfa1", + "name": "audience", "protocol": "openid-connect", - "protocolMapper": "forced-role-mapper", + "protocolMapper": "oidc-audience-mapper", "consentRequired": false, "config": { - "claim.name": "roles", - "jsonType.label": "String", - "key.name": "roles" + "included.client.audience": "ios-app", + "id.token.claim": "false", + "lightweight.claim": "false", + "introspection.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "false" } } ] @@ -3168,8 +3724,8 @@ "role_list", "realm-roles", "audience-opex-api-key", - "user-roles", - "Forced_Roles" + "web-app-force-audience", + "ios-app-force-audience" ], "defaultOptionalClientScopes": [ "offline_access", @@ -3249,13 +3805,13 @@ "subComponents": {}, "config": { "allowed-protocol-mapper-types": [ - "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-property-mapper", - "oidc-usermodel-attribute-mapper", - "oidc-address-mapper", "oidc-full-name-mapper", + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", + "oidc-address-mapper", "saml-role-list-mapper" ] } @@ -3322,14 +3878,14 @@ "subComponents": {}, "config": { "allowed-protocol-mapper-types": [ - "saml-user-property-mapper", + "saml-role-list-mapper", "oidc-sha256-pairwise-sub-mapper", - "oidc-address-mapper", - "oidc-usermodel-attribute-mapper", "saml-user-attribute-mapper", - "saml-role-list-mapper", "oidc-full-name-mapper", - "oidc-usermodel-property-mapper" + "oidc-address-mapper", + "oidc-usermodel-attribute-mapper", + "oidc-usermodel-property-mapper", + "saml-user-property-mapper" ] } } @@ -3454,6 +4010,32 @@ } ] }, + { + "id": "12db0ccb-9df4-4173-95fc-94650a4167e4", + "alias": "Opex Combined Flow", + "description": "", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": false, + "authenticationExecutions": [ + { + "authenticator": "bootstrap-token-grant", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 1, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 2, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, { "id": "cb9a1006-177c-4ae9-8a13-a392cac637f9", "alias": "Opex Direct Grant", @@ -3853,6 +4435,24 @@ } ] }, + { + "id": "8caf9d2f-065a-4da8-96e3-78eca90d399d", + "alias": "opex_login_flow", + "description": "", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": false, + "authenticationExecutions": [ + { + "authenticator": "allow-access-authenticator", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 0, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, { "id": "d3363796-a3f2-458a-9510-cd0daaedbb18", "alias": "registration", @@ -4092,17 +4692,24 @@ "firstBrokerLoginFlow": "first broker login", "attributes": { "cibaBackchannelTokenDeliveryMode": "poll", - "cibaExpiresIn": "120", "cibaAuthRequestedUserHint": "login_hint", - "oauth2DeviceCodeLifespan": "600", "clientOfflineSessionMaxLifespan": "0", "oauth2DevicePollingInterval": "5", "clientSessionIdleTimeout": "0", - "parRequestUriLifespan": "60", - "clientSessionMaxLifespan": "0", + "actionTokenGeneratedByUserLifespan.verify-email": "", + "actionTokenGeneratedByUserLifespan.idp-verify-account-via-email": "", "clientOfflineSessionIdleTimeout": "0", + "actionTokenGeneratedByUserLifespan.execute-actions": "", "cibaInterval": "5", - "realmReusableOtpCode": "false" + "realmReusableOtpCode": "false", + "cibaExpiresIn": "120", + "oauth2DeviceCodeLifespan": "600", + "parRequestUriLifespan": "60", + "clientSessionMaxLifespan": "0", + "frontendUrl": "http://keycloak:8080", + "acr.loa.map": "{}", + "shortVerificationUri": "", + "actionTokenGeneratedByUserLifespan.reset-credentials": "" }, "keycloakVersion": "26.1.5", "userManagedAccessAllowed": false, From b71535be7a56f8556f3ce3bde3e1411a4aa36b12 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:33:38 +0330 Subject: [PATCH 02/22] Update OpexError.kt --- common/src/main/kotlin/co/nilin/opex/common/OpexError.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt b/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt index 6edb3ba9d..27e99a3e5 100644 --- a/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt +++ b/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt @@ -48,7 +48,6 @@ enum class OpexError(val code: Int, val message: String?, val status: HttpStatus AlreadyInKYC(5007, "KYC flow for this user has executed", HttpStatus.BAD_REQUEST), UserKYCBlocked(5008, "User is blocked from KYC", HttpStatus.BAD_REQUEST), InvalidPassword(5009, "Password is not valid", HttpStatus.BAD_REQUEST), - UserAlreadyExists(5009, "User is already registered", HttpStatus.BAD_REQUEST), LoginIsLimited(5010, "Your email is not in whitelist", HttpStatus.BAD_REQUEST), RegisterIsLimited(5011, "Your email is not in whitelist", HttpStatus.BAD_REQUEST), GmailNotFoundInToken(5012, "Email not found in Google token", HttpStatus.NOT_FOUND), @@ -61,6 +60,7 @@ enum class OpexError(val code: Int, val message: String?, val status: HttpStatus InternalIdGenerateFailed(5019, "Internal id generate failed", HttpStatus.INTERNAL_SERVER_ERROR), CaptchaRequired(5020, "Captcha required", HttpStatus.BAD_REQUEST), UsernameOrPasswordIsIncorrect(5021, "Username or password is incorrect", HttpStatus.BAD_REQUEST), + UserAlreadyExists(5022, "User is already registered", HttpStatus.BAD_REQUEST), // code 6000: wallet WalletOwnerNotFound(6001, null, HttpStatus.NOT_FOUND), @@ -229,4 +229,4 @@ enum class OpexError(val code: Int, val message: String?, val status: HttpStatus } } -} \ No newline at end of file +} From 97d2bcd62e795b3e857ce02823f55c0d79071e65 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:05:03 +0330 Subject: [PATCH 03/22] Fix confirm registration service --- .../kotlin/co/nilin/opex/api/core/inout/auth/UserRegister.kt | 2 ++ .../kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt | 2 +- .../src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt | 2 ++ .../main/kotlin/co/nilin/opex/auth/service/RegisterService.kt | 2 ++ .../co/nilin/opex/matching/engine/app/config/AppConfig.kt | 3 ++- 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/auth/UserRegister.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/auth/UserRegister.kt index 917fcd1fd..fbe3e3cff 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/auth/UserRegister.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/auth/UserRegister.kt @@ -25,6 +25,8 @@ data class ConfirmRegisterRequest( val token: String, val clientId: String?, val clientSecret: String?, + val referralCode: String?, + val rememberMe: Boolean? ) : Device() data class TokenData( diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt index ca831aa51..8f702d2a4 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt @@ -107,7 +107,7 @@ class AuthProxyImpl(@Qualifier("generalWebClient") private val webClient: WebCli override suspend fun confirmRegister(request: ConfirmRegisterRequest): Token { return webClient.post() - .uri("$baseUrl/v1/user/public/register/ccofirm") + .uri("$baseUrl/v1/user/public/register/confirm") .accept(MediaType.APPLICATION_JSON) .body(Mono.just(request)) .retrieve() diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt index 88cdb430a..1b5359d0c 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt @@ -24,6 +24,8 @@ data class ConfirmRegisterRequest( val token: String, val clientId: String?, val clientSecret: String?, + val rememberMe: Boolean? = false, + val referralCode: String? ) : Device() data class TokenData( diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt index 4f95d9e40..5d2799d20 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt @@ -81,6 +81,8 @@ class RegisterService( null else { val token = keycloakProxy.getUserToken(username, request.password, request.clientId, request.clientSecret) + if (request.rememberMe == false) + token.refreshToken = null sendLoginEvent(user.id, token.sessionState, request, token.expiresIn) return token } diff --git a/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/config/AppConfig.kt b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/config/AppConfig.kt index 246c936ee..85808b5c8 100644 --- a/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/config/AppConfig.kt +++ b/matching-engine/matching-engine-app/src/main/kotlin/co/nilin/opex/matching/engine/app/config/AppConfig.kt @@ -17,7 +17,8 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @Configuration -class AppConfig { +class AppConfig( +) { @Autowired private lateinit var symbols: List From 29461170ce9ab0f6dddf84a4a12690ae44c10940 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:17:41 +0330 Subject: [PATCH 04/22] Hot fix: Refactor transfer logic to use sourceAmount --- .../co/nilin/opex/wallet/app/service/TransferService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt index 49087cf13..f42d4bbd5 100644 --- a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt +++ b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt @@ -260,7 +260,7 @@ class TransferService( val withdrawRef = "${refPrefix}${reserveNumber}-withdraw" val depositRef = "${refPrefix}${reserveNumber}-deposit" - val senderTransfer = _transfer( + val senderTransfer = ( symbol = reservations.sourceSymbol, senderWalletType = reservations.senderWalletType, senderUuid = reservations.senderUuid, @@ -271,10 +271,10 @@ class TransferService( transferRef = withdrawRef, transferCategory = transferCategory, destSymbol = reservations.destSymbol, - destAmount = reservations.reservedDestAmount + destAmount = reservations.sourceAmount ).transferResult - val receiverTransfer = _transfer( + val receiverTransfer = ( symbol = reservations.destSymbol, senderWalletType = WalletType.MAIN, senderUuid = walletOwnerManager.systemUuid, @@ -454,4 +454,4 @@ class TransferService( throw OpexError.InvalidMaximumAmount.exception("amount is higher than maximum") } } -} \ No newline at end of file +} From c40d7fa5fa9c3401c26afbc0e72d4f068b9b40b9 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:44:30 +0330 Subject: [PATCH 05/22] Hot fix: Refactor transfer logic to use sourceAmount --- .../co/nilin/opex/wallet/app/service/TransferService.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt index f42d4bbd5..5f747c129 100644 --- a/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt +++ b/wallet/wallet-app/src/main/kotlin/co/nilin/opex/wallet/app/service/TransferService.kt @@ -260,7 +260,7 @@ class TransferService( val withdrawRef = "${refPrefix}${reserveNumber}-withdraw" val depositRef = "${refPrefix}${reserveNumber}-deposit" - val senderTransfer = ( + val senderTransfer = _transfer( symbol = reservations.sourceSymbol, senderWalletType = reservations.senderWalletType, senderUuid = reservations.senderUuid, @@ -270,11 +270,11 @@ class TransferService( description = description, transferRef = withdrawRef, transferCategory = transferCategory, - destSymbol = reservations.destSymbol, + destSymbol = reservations.sourceSymbol, destAmount = reservations.sourceAmount ).transferResult - val receiverTransfer = ( + val receiverTransfer = _transfer( symbol = reservations.destSymbol, senderWalletType = WalletType.MAIN, senderUuid = walletOwnerManager.systemUuid, From 5fe6e064ed2afa209dcc69a4df06b75dfd9db6da Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:22:53 +0330 Subject: [PATCH 06/22] Change the return type of otp providers (#701) --- .../repository/SMSProviderRouteRepository.kt | 4 ++-- .../opex/otp/app/service/SMSProviderRouter.kt | 7 +++++-- .../app/conttroller/SMSProviderRouterTest.kt | 21 ++++++++++--------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/repository/SMSProviderRouteRepository.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/repository/SMSProviderRouteRepository.kt index 2a51d6e6d..368f1f834 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/repository/SMSProviderRouteRepository.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/repository/SMSProviderRouteRepository.kt @@ -3,9 +3,9 @@ package co.nilin.opex.otp.app.repository import co.nilin.opex.otp.app.model.SMSProviderRoute import org.springframework.data.repository.kotlin.CoroutineCrudRepository import org.springframework.stereotype.Repository +import reactor.core.publisher.Flux @Repository interface SMSProviderRouteRepository : CoroutineCrudRepository { - - fun findAllByEnabledTrue(): List + suspend fun findAllByEnabledTrue(): Flux } \ No newline at end of file diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/SMSProviderRouter.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/SMSProviderRouter.kt index 143c41e74..648867e91 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/SMSProviderRouter.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/SMSProviderRouter.kt @@ -3,6 +3,7 @@ package co.nilin.opex.otp.app.service; import co.nilin.opex.otp.app.data.SMSProviderType; import co.nilin.opex.otp.app.proxy.SMSProvider; import co.nilin.opex.otp.app.repository.SMSProviderRouteRepository +import kotlinx.coroutines.reactor.awaitSingleOrNull import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component; @@ -20,8 +21,10 @@ class SMSProviderRouter( val routes = routeRepository.findAllByEnabledTrue() val providerType = routes - .sortedByDescending { it.prefix.length } - .firstOrNull { receiver.startsWith(it.prefix) } + .collectList() + .awaitSingleOrNull() + ?.sortedByDescending { it.prefix.length } + ?.firstOrNull { receiver.startsWith(it.prefix) } ?.provider ?.let(SMSProviderType::valueOf) ?: defaultSmsProvider diff --git a/otp/otp-app/src/test/kotlin/co/nilin/opex/otp/app/conttroller/SMSProviderRouterTest.kt b/otp/otp-app/src/test/kotlin/co/nilin/opex/otp/app/conttroller/SMSProviderRouterTest.kt index b26b2a21c..1251f7f42 100644 --- a/otp/otp-app/src/test/kotlin/co/nilin/opex/otp/app/conttroller/SMSProviderRouterTest.kt +++ b/otp/otp-app/src/test/kotlin/co/nilin/opex/otp/app/conttroller/SMSProviderRouterTest.kt @@ -9,6 +9,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.mockito.Mockito import org.mockito.Mockito.mock +import reactor.core.publisher.Flux class SMSProviderRouterTest { @@ -41,9 +42,9 @@ class SMSProviderRouterTest { fun givenNoMatchingRoute_whenGetProvider_thenDefaultProviderReturned(): Unit = runBlocking { Mockito.`when`(repository.findAllByEnabledTrue()) .thenReturn( - listOf( - SMSProviderRoute(1,"+98", SMSProviderType.SMSTO.name), - SMSProviderRoute(2,"+989", SMSProviderType.KAVENEGAR.name), + Flux.just( + SMSProviderRoute(1, "+98", SMSProviderType.SMSTO.name), + SMSProviderRoute(2, "+989", SMSProviderType.KAVENEGAR.name), ) ) val provider = router.getProvider("+447700123456") @@ -54,9 +55,9 @@ class SMSProviderRouterTest { fun givenNoMatchingRoute_whenGetProvider_thenLongestMatchedProviderReturned(): Unit = runBlocking { Mockito.`when`(repository.findAllByEnabledTrue()) .thenReturn( - listOf( - SMSProviderRoute(1,"+98", SMSProviderType.SMSTO.name), - SMSProviderRoute(2,"+989", SMSProviderType.KAVENEGAR.name), + Flux.just( + SMSProviderRoute(1, "+98", SMSProviderType.SMSTO.name), + SMSProviderRoute(2, "+989", SMSProviderType.KAVENEGAR.name), ) ) val provider = router.getProvider("+989556677788") @@ -67,10 +68,10 @@ class SMSProviderRouterTest { fun givenNoMatchingRoute_whenGetProvider_thenMatchedProviderReturned(): Unit = runBlocking { Mockito.`when`(repository.findAllByEnabledTrue()) .thenReturn( - listOf( - SMSProviderRoute(1,"+98", SMSProviderType.SMSTO.name), - SMSProviderRoute(2,"+989", SMSProviderType.KAVENEGAR.name), - SMSProviderRoute(3,"+44", SMSProviderType.TWILIO.name), + Flux.just( + SMSProviderRoute(1, "+98", SMSProviderType.SMSTO.name), + SMSProviderRoute(2, "+989", SMSProviderType.KAVENEGAR.name), + SMSProviderRoute(3, "+44", SMSProviderType.TWILIO.name), ) ) From ab2767ec45e71115776af3156f5ba306a87115e2 Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:45:19 +0330 Subject: [PATCH 07/22] Add smsir to sms providers (#704) --- .../opex/otp/app/data/SMSProviderType.kt | 3 +- .../co/nilin/opex/otp/app/proxy/SMSIRProxy.kt | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/data/SMSProviderType.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/data/SMSProviderType.kt index 62c092aa1..f38a971bb 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/data/SMSProviderType.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/data/SMSProviderType.kt @@ -3,5 +3,6 @@ package co.nilin.opex.otp.app.data enum class SMSProviderType { KAVENEGAR, TWILIO, - SMSTO + SMSTO, + SMSIR } \ No newline at end of file diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt new file mode 100644 index 000000000..10156b8cd --- /dev/null +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt @@ -0,0 +1,55 @@ +package co.nilin.opex.otp.app.proxy + +import co.nilin.opex.common.OpexError +import co.nilin.opex.common.utils.LoggerDelegate +import co.nilin.opex.otp.app.data.SMSProviderType +import co.nilin.opex.otp.app.repository.SMSProviderRepository +import kotlinx.coroutines.reactor.awaitSingleOrNull +import org.springframework.http.MediaType +import org.springframework.stereotype.Component +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.bodyToMono +import org.springframework.web.util.UriComponentsBuilder + +@Component +class SMSIRProxy( + private val webClient: WebClient, + private val smsProviderRepository: SMSProviderRepository, +) : SMSProvider { + + override val type = SMSProviderType.SMSIR + + + private val logger by LoggerDelegate() + + override suspend fun send(receiver: String, message: String): Boolean { + val config = smsProviderRepository.findById(type.name) + ?: throw OpexError.UnableToSendOTP.exception() + + val uri = UriComponentsBuilder + .fromUriString("${config.baseUrl}/v1/send") + .queryParam("username", config.username) + .queryParam("password", config.password) + .queryParam("mobile", receiver) + .queryParam("line", config.sender) + .queryParam("text", "otp code : $message") + .build(true) + .toUri() + + return try { + val response = webClient.get() + .uri(uri) + .accept(MediaType.TEXT_PLAIN) + .retrieve() + .onStatus({ it.isError }) { it.createException() } + .bodyToMono() + .awaitSingleOrNull() + + logger.debug("Message sent to receiver $receiver. Response: $response") + true + } catch (e: Exception) { + logger.error("Failed to send SMS", e) + false + } + } +} \ No newline at end of file From c9f7dbac36ceed406ba3866cb6d7fbd6cd103dc0 Mon Sep 17 00:00:00 2001 From: fatemeh imanipour Date: Mon, 3 Aug 2026 17:19:06 +0330 Subject: [PATCH 08/22] Fix the vault configuration of matching gateway --- .../matching-gateway-app/src/main/resources/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matching-gateway/matching-gateway-app/src/main/resources/application.yml b/matching-gateway/matching-gateway-app/src/main/resources/application.yml index 860504b23..9d7103666 100644 --- a/matching-gateway/matching-gateway-app/src/main/resources/application.yml +++ b/matching-gateway/matching-gateway-app/src/main/resources/application.yml @@ -44,6 +44,8 @@ spring: instance-id: ${spring.application.name}:${server.port} healthCheckInterval: 20s prefer-ip-address: true + config: + import: vault://secret/${spring.application.name} management: endpoints: web: From ee6587dcab44537c3882a757e69f8bbcae6e862d Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:08:46 +0330 Subject: [PATCH 09/22] Hot fix : update SMSIRProxy (#706) --- .../main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt index 10156b8cd..7c9074b8d 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt @@ -32,8 +32,8 @@ class SMSIRProxy( .queryParam("password", config.password) .queryParam("mobile", receiver) .queryParam("line", config.sender) - .queryParam("text", "otp code : $message") - .build(true) + .queryParam("text", "Your OTP code is $message") + .build() .toUri() return try { @@ -52,4 +52,4 @@ class SMSIRProxy( false } } -} \ No newline at end of file +} From 2af23cf3adce0b2ff43d6c934e310c00bb793cbc Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:41:32 +0330 Subject: [PATCH 10/22] Fix conflicts (#708) --- README.md | 152 +++++++++++++++--- docker-compose.local.yml | 26 ++- .../co/nilin/opex/otp/app/proxy/SMSIRProxy.kt | 2 +- .../postgres/imp/ProfileManagementImp.kt | 2 +- 4 files changed, 154 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c10eb526a..e0ed51618 100644 --- a/README.md +++ b/README.md @@ -51,43 +51,145 @@ You need to have [Maven](https://maven.apache.org) and [Docker](https://www.dock ## Environment Variables ``` -APP_NAME=Opex-local -APP_BASE_URL=localhost:8080 +APP_NAME=Opex +APP_BASE_URL=http://localhost:8094 PANEL_PASS=admin BACKEND_USER=admin -KEYCLOAK_ADMIN_USERNAME=opex -KEYCLOAK_ADMIN_PASSWORD=hiopex -SMTP_PASSWORD=x +SMTP_PASS=x +OPEX_ADMIN_KEYCLOAK_CLIENT_SECRET=x API_KEY_CLIENT_SECRET=x -KEYCLOAK_FRONTEND_URL=http://localhost:8083/auth -KEYCLOAK_ADMIN_URL=http://localhost:8083/auth +KEYCLOAK_FRONTEND_URL=http://localhost:8193 +KEYCLOAK_ADMIN_URL=http://localhost:8193/ KEYCLOAK_VERIFY_REDIRECT_URL=http://localhost:8080/verify KEYCLOAK_FORGOT_REDIRECT_URL=http://localhost:8080/forgot +PREFERENCES=preferences.yml +LOGSTASH_ELASTIC_USER=x +LOGSTASH_ELASTIC_PASSWORD=x +KIBANA_ELASTIC_USER=x +KIBANA_ELASTIC_PASSWORD=x +GRAFANA_PASSWORD=x +DRIVE_FOLDER_ID=x +STORAGE_FOLDER_ID=x +WALLET_BACKUP_ENABLED_STORAGE=false +WALLET_BACKUP_ENABLED_GOOGLE_DRIVE=false WHITELIST_REGISTER_ENABLED=true WHITELIST_LOGIN_ENABLED=true -WALLET_BACKUP_ENABLED=false -OPEX_ADMIN_KEYCLOAK_CLIENT_SECRET=x +VANDAR_API_KEY=x TAG=debug +KC_DB_USERNAME=admin +KC_DB_PASSWORD=admin +KC_PANEL_USERNAME=admin +KC_PANEL_PASSWORD=admin +DB_USER=opex +DB_PASS=hiopex +SMTP_HOST=x +SMTP_USER=x +SMTP_PASS=x +CLIENT_ID=X +CLIENT_SECRET=X +SMS_PROVIDER_API_KEY=x +KC_GOOGLE_CLIENT_ID=x +KC_GOOGLE_CLIENT_SECRET=*** +KC_ADMIN_CLIENT_SECRET=x +KAFKA_CLUSTER_ID=x +JIBIT_API_KEY=x +JIBIT_SECRET_KEY=x +JIBIT_URL=x +WITHDRAW_LIMIT_ENABLED=true +WITHDRAW_OTP_REQUIRED_COUNT=0 +WITHDRAW_BANK_ACCOUNT_VALIDATION=false +TRADE_VOLUME_CALCULATION_CURRENCY= USDT +WITHDRAW_VOLUME_CALCULATION_CURRENCY= USDT +TOTAL_ASSET_CALCULATION_CURRENCY= USDT +ADMIN_APPROVAL_PROFILE_COMPLETION_REQUEST=true +MOBILE_IDENTITY_INQUIRY= false +PERSONAL_IDENTITY_INQUIRY= false +ADMIN_APPROVAL_BANK_ACCOUNT=false +KC_PRE_AUTH_CLIENT_SECRET= +KC_ISSUER_URL=http://keycloak:8080/realms/opex +OTP_CODE_RESPONSE_ENABLED=true +CAPTCHA_ENABLED=false +JWK_ENDPOINT=x +SMTP_PROXY_HOST=x +SMTP_PROXY_PORT=x +SMTP_PROXY_ENABLED=false +SWAGGER_API_DOCS_ENABLED="true" +SWAGGER_UI_ENABLED="true" +SWAGGER_AUTH_ENABLED="false" +SWAGGER_AUTH_AUTHORITY="ROLE_admin" ``` -| Variable | Description | -|:--------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| APP_NAME
APP_BASE_URL | Application metadata used by Keycloak | -| PANEL_PASS | Vault's admin panel password | -| BACKEND_USER | Username used by services to access vault data. Also used as the username for Vault admin panel | -| KEYCLOAK_ADMIN_USERNAME | Username for Keycloak admin panel | -| KEYCLOAK_ADMIN_PASSWORD | Password for Keycloak admin panel | -| SMTP_PASSWORD | SMTP password used by keycloak to send emails for various operations (e.g. user verification, reset password) | -| API_KEY_CLIENT_SECRET | In order to access the api key feature, please follow the steps below:
1. Go to Keycloak admin panel located at http://localhost:8083/auth/admin/master/console/#/realms/opex/clients
2. Login with the username and password you provided in the `.env` file (KEYCLOAK_ADMIN_USERNAME and KEYCLOAK_ADMIN_PASSWORD)
3. Go to `clients` section on the left menu
4. Click on `opex-api-key` client
5. In the credentials section, click on `Regenerate Secret` button
6. Copy the generated secret and paste it into this section | -| KEYCLOAK_FRONTEND_URL
KEYCLOAK_ADMIN_URL
KEYCLOAK_VERIFY_REDIRECT_URL
KEYCLOAK_FORGOT_REDIRECT_URL | Replace `localhost` with your server's IP if you're not running on local machine. Do not change the rest. | -| WHITELIST_REGISTER_ENABLED | Allows registration only for whitelisted emails | -| WHITELIST_LOGIN_ENABLED | Allows login only for whitelisted emails | -| WALLET_BACKUP_ENABLED | Enables wallet data backup to google drive folder. In order to use this feature, you need to have `drive-key.json` file (obtained from google drive API panel) in the root directory of project | -| OPEX_ADMIN_KEYCLOAK_CLIENT_SECRET | Ignore this for now. Will be removed soon | -| Tag | Labels locally built docker images | +| Variable | Description | +| :------------------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------- | +| `APP_NAME` | Application name. | +| `APP_BASE_URL` | Base URL of the application. | +| `PANEL_PASS` | Password for the Vault admin panel. | +| `BACKEND_USER` | Username used by backend services to access Vault data and the Vault admin panel. | +| `SMTP_PASS` | Password used by Keycloak to authenticate with the SMTP server for sending emails such as verification and password reset emails. | +| `OPEX_ADMIN_KEYCLOAK_CLIENT_SECRET` | Ignore this for now. Will be removed soon. | +| `API_KEY_CLIENT_SECRET` | Secret used to access the API key functionality. | +| `KEYCLOAK_FRONTEND_URL` | URL used by the frontend to communicate with Keycloak. | +| `KEYCLOAK_ADMIN_URL` | URL used to access the Keycloak admin panel. | +| `KEYCLOAK_VERIFY_REDIRECT_URL` | Redirect URL used after successful email verification. | +| `KEYCLOAK_FORGOT_REDIRECT_URL` | Redirect URL used for the forgot-password flow. | +| `PREFERENCES` | Path to the application's preferences configuration file. | +| `LOGSTASH_ELASTIC_USER` | Username used by Logstash to authenticate with Elasticsearch. | +| `LOGSTASH_ELASTIC_PASSWORD` | Password used by Logstash to authenticate with Elasticsearch. | +| `KIBANA_ELASTIC_USER` | Username used by Kibana to authenticate with Elasticsearch. | +| `KIBANA_ELASTIC_PASSWORD` | Password used by Kibana to authenticate with Elasticsearch. | +| `GRAFANA_PASSWORD` | Password used to access Grafana. | +| `DRIVE_FOLDER_ID` | Google Drive folder ID used for storing backups. | +| `STORAGE_FOLDER_ID` | Storage folder ID used for storing backups. | +| `WALLET_BACKUP_ENABLED_STORAGE` | Enables or disables wallet backup to the configured storage. | +| `WALLET_BACKUP_ENABLED_GOOGLE_DRIVE` | Enables or disables wallet backup to Google Drive. | +| `WHITELIST_REGISTER_ENABLED` | Allows registration only for users whose email addresses are whitelisted. | +| `WHITELIST_LOGIN_ENABLED` | Allows login only for users whose email addresses are whitelisted. | +| `VANDAR_API_KEY` | API key used to authenticate with the Vandar service. | +| `TAG` | Tag used for locally built Docker images. | +| `KC_DB_USERNAME` | Username used by Keycloak to connect to its database. | +| `KC_DB_PASSWORD` | Password used by Keycloak to connect to its database. | +| `KC_PANEL_USERNAME` | Username used to access the Keycloak admin panel. | +| `KC_PANEL_PASSWORD` | Password used to access the Keycloak admin panel. | +| `DB_USER` | Username used by the application to connect to the database. | +| `DB_PASS` | Password used by the application to connect to the database. | +| `SMTP_HOST` | Hostname or IP address of the SMTP server. | +| `SMTP_USER` | Username used to authenticate with the SMTP server. | +| `CLIENT_ID` | Client ID used for authentication with the configured external service. | +| `CLIENT_SECRET` | Client secret used together with `CLIENT_ID` for authentication. | +| `SMS_PROVIDER_API_KEY` | API key used to authenticate with the configured SMS provider. | +| `KC_GOOGLE_CLIENT_ID` | Google OAuth client ID used by Keycloak for Google authentication. | +| `KC_GOOGLE_CLIENT_SECRET` | Google OAuth client secret used by Keycloak for Google authentication. | +| `KC_ADMIN_CLIENT_SECRET` | Client secret used to authenticate with the Keycloak admin API. | +| `KAFKA_CLUSTER_ID` | Identifier of the Kafka cluster used by the application. | +| `JIBIT_API_KEY` | API key used to authenticate with the Jibit service. | +| `JIBIT_SECRET_KEY` | Secret key used together with the Jibit API key for Jibit authentication. | +| `JIBIT_URL` | Base URL of the Jibit API. | +| `WITHDRAW_LIMIT_ENABLED` | Enables or disables withdrawal limits. | +| `WITHDRAW_OTP_REQUIRED_COUNT` | Number of withdrawal requests after which OTP verification is required. | +| `WITHDRAW_BANK_ACCOUNT_VALIDATION` | Enables or disables bank account validation during withdrawals. | +| `TRADE_VOLUME_CALCULATION_CURRENCY` | Currency used to calculate trading volume. | +| `WITHDRAW_VOLUME_CALCULATION_CURRENCY` | Currency used to calculate withdrawal volume. | +| `TOTAL_ASSET_CALCULATION_CURRENCY` | Currency used to calculate the total value of user assets. | +| `ADMIN_APPROVAL_PROFILE_COMPLETION_REQUEST` | Determines whether profile completion requests require admin approval. | +| `MOBILE_IDENTITY_INQUIRY` | Enables or disables mobile identity verification. | +| `PERSONAL_IDENTITY_INQUIRY` | Enables or disables personal identity verification. | +| `ADMIN_APPROVAL_BANK_ACCOUNT` | Determines whether adding or updating a bank account requires admin approval. | +| `KC_PRE_AUTH_CLIENT_SECRET` | Client secret used by the Keycloak pre-authentication client. | +| `KC_ISSUER_URL` | Internal Keycloak issuer URL used by backend services for authentication and token validation. | +| `OTP_CODE_RESPONSE_ENABLED` | Enables or disables returning the OTP code in the API response. | +| `CAPTCHA_ENABLED` | Enables or disables CAPTCHA validation. | +| `JWK_ENDPOINT` | Endpoint used to retrieve JSON Web Keys (JWKs) for JWT validation. | +| `SMTP_PROXY_HOST` | Hostname or IP address of the SMTP proxy. | +| `SMTP_PROXY_PORT` | Port of the SMTP proxy. | +| `SMTP_PROXY_ENABLED` | Enables or disables the SMTP proxy. | +| `SWAGGER_API_DOCS_ENABLED` | Enables or disables Swagger/OpenAPI API documentation. | +| `SWAGGER_UI_ENABLED` | Enables or disables the Swagger UI. | +| `SWAGGER_AUTH_ENABLED` | Enables or disables authentication for Swagger UI. | +| `SWAGGER_AUTH_AUTHORITY` | Required authority/role for accessing Swagger UI when authentication is enabled. | + ## Live Demo -Deployed at [app.opex.dev](https://app.opex.dev). +Deployed at [beta.opex.dev](https://beta.opex.dev). ## Architecture Overview diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 3ab397629..bcc4ed564 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -36,9 +36,33 @@ services: postgres-otp: ports: - "127.0.0.1:5462:5432" + postgres-accountant: + ports: + - "5432:5432" + postgres-eventlog: + ports: + - "5433:5432" + postgres-auth: + ports: + - "5434:5432" + postgres-wallet: + ports: + - "5435:5432" + postgres-api: + ports: + - "5436:5432" postgres-market: ports: - - "127.0.0.1:5438:5432" + - "5438:5432" + postgres-bc-gateway: + ports: + - "5437:5432" + postgres-matching-gateway: + ports: + - "5439:5432" + postgres-profile: + ports: + - "5440:5432" accountant: ports: - "127.0.0.1:8089:8080" diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt index 7c9074b8d..df0f0bbdd 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/proxy/SMSIRProxy.kt @@ -52,4 +52,4 @@ class SMSIRProxy( false } } -} +} \ No newline at end of file diff --git a/profile/profile-ports/profile-postgres/src/main/kotlin/co/nilin/opex/profile/ports/postgres/imp/ProfileManagementImp.kt b/profile/profile-ports/profile-postgres/src/main/kotlin/co/nilin/opex/profile/ports/postgres/imp/ProfileManagementImp.kt index 339fef496..7dccf0fa1 100644 --- a/profile/profile-ports/profile-postgres/src/main/kotlin/co/nilin/opex/profile/ports/postgres/imp/ProfileManagementImp.kt +++ b/profile/profile-ports/profile-postgres/src/main/kotlin/co/nilin/opex/profile/ports/postgres/imp/ProfileManagementImp.kt @@ -196,7 +196,7 @@ class ProfileManagementImp( } override suspend fun validateMobileForUpdate(userId: String, mobile: String) { - validateMobileFormat(mobile) +// validateMobileFormat(mobile) val profile = profileRepository.findByUserId(userId)?.awaitFirstOrNull() ?: throw OpexError.ProfileNotfound.exception() From 5c7cee0c80a8916bcc10f6bd14aad3d7ed7136e1 Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:22:04 +0330 Subject: [PATCH 11/22] Support pair categories and chart flags (#709) --- .../nilin/opex/api/core/inout/PairCategory.kt | 7 ++ .../opex/api/core/inout/PairInfoResponse.kt | 3 + .../nilin/opex/api/core/inout/PairSetting.kt | 3 + .../ports/opex/controller/MarketController.kt | 41 ++++++++-- .../gateway/app/service/sample/Samples.kt | 2 +- .../postgres/dao/PairCategoryRepository.kt | 14 ++++ .../postgres/dao/PairSettingRepository.kt | 12 ++- .../gateway/ports/postgres/dto/PairSetting.kt | 10 ++- .../postgres/impl/PairSettingServiceImpl.kt | 77 +++++++++++++++---- .../ports/postgres/model/PairCategory.kt | 7 ++ .../ports/postgres/model/PairCategoryModel.kt | 11 +++ .../ports/postgres/model/PairSettingModel.kt | 14 +++- .../service/PairSettingInitializer.kt | 4 +- .../gateway/ports/postgres/util/Convertor.kt | 3 + .../src/main/resources/schema.sql | 21 ++++- 15 files changed, 194 insertions(+), 35 deletions(-) create mode 100644 api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt create mode 100644 matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairCategoryRepository.kt create mode 100644 matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategory.kt create mode 100644 matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategoryModel.kt diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt new file mode 100644 index 000000000..b6b92ce16 --- /dev/null +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt @@ -0,0 +1,7 @@ +package co.nilin.opex.api.core.inout + +enum class PairCategory { + REAL_ASSET_TOKEN, + FIAT, + CRYPTO +} \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt index d6c00fe0e..1fed02103 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt @@ -10,4 +10,7 @@ data class PairInfoResponse( val minOrder : BigDecimal, val maxOrder : BigDecimal, val orderTypes : String, + val internalChart: Boolean, + val globalChart: Boolean, + val categories: List = emptyList() ) \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt index b3d7d5946..782a09a0e 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt @@ -10,4 +10,7 @@ class PairSetting( val maxOrder : BigDecimal, val orderTypes : String, val updateDate: LocalDateTime? = null, + val internalChart: Boolean, + val globalChart: Boolean, + val categories: List = emptyList() ) \ No newline at end of file diff --git a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt index 290c55827..6ce743d62 100644 --- a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt +++ b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt @@ -18,11 +18,7 @@ import io.swagger.v3.oas.annotations.tags.Tag import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.springframework.beans.factory.annotation.Value -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.RestController +import org.springframework.web.bind.annotation.* import java.math.BigDecimal import java.time.ZoneId @@ -104,12 +100,45 @@ Response body: isAvailable = isAvailable, minOrder = minOrder, maxOrder = maxOrder, - orderTypes = orderTypes + orderTypes = orderTypes, + internalChart = internalChart, + globalChart = globalChart, + categories = categories ) } } } + @GetMapping("/pair/categories") + @Operation( + summary = "Get trading pair categories", + description = """ +Security: +- Public endpoint. No Bearer token is required. + +Source of values: +- Category names returned here can be used by clients to filter pairs. + +Response body: +- Array of pair category names. + """, + responses = [ + ApiResponse( + responseCode = "200", + description = "Trading pair categories returned successfully.", + content = [ + Content( + mediaType = "application/json", + array = ArraySchema(schema = Schema(type = "string")) + ) + ] + ) + ] + ) + fun getPairCategories(): List { + return PairCategory.entries.map { it.name } + } + @GetMapping("/chain") @Operation( summary = "Get chains", diff --git a/matching-gateway/matching-gateway-app/src/test/kotlin/co/nilin/opex/matching/gateway/app/service/sample/Samples.kt b/matching-gateway/matching-gateway-app/src/test/kotlin/co/nilin/opex/matching/gateway/app/service/sample/Samples.kt index ea9f1ab47..6f14f8ade 100644 --- a/matching-gateway/matching-gateway-app/src/test/kotlin/co/nilin/opex/matching/gateway/app/service/sample/Samples.kt +++ b/matching-gateway/matching-gateway-app/src/test/kotlin/co/nilin/opex/matching/gateway/app/service/sample/Samples.kt @@ -24,7 +24,7 @@ object VALID { val PAIR_CONFIG = PairConfig(ETH_USDT, ETH, USDT, BigDecimal.valueOf(0.01), BigDecimal.valueOf(0.0001)) - val PAIR_SETTING = PairSetting(ETH_USDT, true, 0.0000001.toBigDecimal(), 100.toBigDecimal(), "LIMIT_ORDER,MARKET_ORDER", null) + val PAIR_SETTING = PairSetting(ETH_USDT, true, 0.0000001.toBigDecimal(), 100.toBigDecimal(), "LIMIT_ORDER,MARKET_ORDER", null,true,true) val CREATE_ORDER_REQUEST_ASK = CreateOrderRequest( UUID, diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairCategoryRepository.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairCategoryRepository.kt new file mode 100644 index 000000000..dd3c39a39 --- /dev/null +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairCategoryRepository.kt @@ -0,0 +1,14 @@ +package co.nilin.opex.matching.gateway.ports.postgres.dao + +import co.nilin.opex.matching.gateway.ports.postgres.model.PairCategoryModel +import kotlinx.coroutines.flow.Flow +import org.springframework.data.repository.kotlin.CoroutineCrudRepository +import org.springframework.stereotype.Repository +import reactor.core.publisher.Mono + +@Repository +interface PairCategoryRepository : CoroutineCrudRepository { + fun findByPair(pair: String): Flow + + fun deleteByPair(pair: String): Mono +} \ No newline at end of file diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairSettingRepository.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairSettingRepository.kt index 12fcff723..dc562879a 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairSettingRepository.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dao/PairSettingRepository.kt @@ -11,6 +11,14 @@ import java.math.BigDecimal interface PairSettingRepository : ReactiveCrudRepository { fun findByPair(pair: String): Mono - @Query("insert into pair_setting(pair,is_available,min_order,max_order,order_types) values(:pair,:isAvailable,:minOrder,:maxOrder,:orderTypes) ") - fun insert(pair: String, isAvailable: Boolean , minOrder : BigDecimal, maxOrder : BigDecimal,orderTypes : String): Mono + @Query("insert into pair_setting(pair,is_available,min_order,max_order,order_types,internal_chart,global_chart) values(:pair,:isAvailable,:minOrder,:maxOrder,:orderTypes,:internalChart,:globalChart) ") + fun insert( + pair: String, + isAvailable: Boolean, + minOrder: BigDecimal, + maxOrder: BigDecimal, + orderTypes: String, + internalChart: Boolean, + globalChart: Boolean + ): Mono } diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dto/PairSetting.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dto/PairSetting.kt index 1fd568629..d13a53b62 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dto/PairSetting.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/dto/PairSetting.kt @@ -1,13 +1,17 @@ package co.nilin.opex.matching.gateway.ports.postgres.dto +import co.nilin.opex.matching.gateway.ports.postgres.model.PairCategory import java.math.BigDecimal import java.time.LocalDateTime class PairSetting( val pair: String, val isAvailable: Boolean, - val minOrder : BigDecimal, - val maxOrder : BigDecimal, - val orderTypes : String, + val minOrder: BigDecimal, + val maxOrder: BigDecimal, + val orderTypes: String, val updateDate: LocalDateTime? = null, + val internalChart: Boolean, + val globalChart: Boolean, + val categories: List = emptyList() ) \ No newline at end of file diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/impl/PairSettingServiceImpl.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/impl/PairSettingServiceImpl.kt index 288ef97c9..0c603e74a 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/impl/PairSettingServiceImpl.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/impl/PairSettingServiceImpl.kt @@ -2,10 +2,14 @@ package co.nilin.opex.matching.gateway.ports.postgres.impl import co.nilin.opex.common.OpexError import co.nilin.opex.common.utils.CacheManager +import co.nilin.opex.matching.gateway.ports.postgres.dao.PairCategoryRepository import co.nilin.opex.matching.gateway.ports.postgres.dao.PairSettingRepository import co.nilin.opex.matching.gateway.ports.postgres.dto.PairSetting +import co.nilin.opex.matching.gateway.ports.postgres.model.PairCategoryModel import co.nilin.opex.matching.gateway.ports.postgres.service.PairSettingService import co.nilin.opex.matching.gateway.ports.postgres.util.toPairSetting +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.reactive.awaitFirst import kotlinx.coroutines.reactive.awaitFirstOrNull import org.springframework.beans.factory.annotation.Qualifier @@ -16,14 +20,20 @@ import java.util.concurrent.TimeUnit @Service class PairSettingServiceImpl( private val pairSettingRepository: PairSettingRepository, + private val pairCategoryRepository: PairCategoryRepository, @Qualifier("appCacheManager") private val cacheManager: CacheManager ) : PairSettingService { override suspend fun load(pair: String): PairSetting { return cacheManager.get("pair-setting:$pair") ?: pairSettingRepository.findByPair(pair) - .awaitFirstOrNull() - ?.let { + .awaitFirstOrNull()?.let { pairSettingModel -> + val categories = pairCategoryRepository.findByPair(pairSettingModel.pair) + .map { it.category } + .toList() + pairSettingModel.categories = categories + pairSettingModel + }?.let { it.toPairSetting().also { cacheManager.put( "pair-setting:${it.pair}", @@ -36,28 +46,61 @@ class PairSettingServiceImpl( } override suspend fun loadAll(): List { - return pairSettingRepository.findAll() - .map { it.toPairSetting() } - .collectList() - .awaitFirstOrNull() ?: emptyList() + val pairSettings = pairSettingRepository.findAll().collectList().awaitFirst() + + if (pairSettings.isEmpty()) { + return emptyList() + } + + val categoriesByPair = pairCategoryRepository.findAll() + .toList() + .groupBy( + keySelector = { it.pair }, + valueTransform = { it.category } + ) + + return pairSettings.map { ps -> + ps.categories = categoriesByPair[ps.pair] ?: emptyList() + ps.toPairSetting() + } } override suspend fun update(pairSetting: PairSetting): PairSetting { - val pairSetting = - pairSettingRepository.findByPair(pairSetting.pair).awaitFirstOrNull() - ?: throw OpexError.PairNotFound.exception() - pairSetting.apply { - this.isAvailable = pairSetting.isAvailable - this.minOrder = pairSetting.minOrder - this.maxOrder = pairSetting.maxOrder - this.orderTypes = pairSetting.orderTypes - this.updateDate = LocalDateTime.now() + val existing = pairSettingRepository.findByPair(pairSetting.pair) + .awaitFirstOrNull() + ?: throw OpexError.PairNotFound.exception() + + existing.apply { + isAvailable = pairSetting.isAvailable + minOrder = pairSetting.minOrder + maxOrder = pairSetting.maxOrder + orderTypes = pairSetting.orderTypes + updateDate = LocalDateTime.now() + internalChart = pairSetting.internalChart + globalChart = pairSetting.globalChart + } + + val saved = pairSettingRepository.save(existing) + .awaitFirst() + + pairCategoryRepository.deleteByPair(pairSetting.pair).awaitFirstOrNull() + pairSetting.categories.forEach { category -> + pairCategoryRepository.save( + PairCategoryModel( + pair = pairSetting.pair, + category = category + ) + ) } - return pairSettingRepository.save(pairSetting).awaitFirst().toPairSetting().also { + + return saved.apply { + categories = pairSetting.categories + }.toPairSetting().also { cacheManager.put( "pair-setting:${it.pair}", it, - 5, TimeUnit.MINUTES + 5, + TimeUnit.MINUTES ) } } diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategory.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategory.kt new file mode 100644 index 000000000..a3e2ad3f0 --- /dev/null +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategory.kt @@ -0,0 +1,7 @@ +package co.nilin.opex.matching.gateway.ports.postgres.model + +enum class PairCategory { + REAL_ASSET_TOKEN, + FIAT, + CRYPTO +} \ No newline at end of file diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategoryModel.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategoryModel.kt new file mode 100644 index 000000000..7203802c9 --- /dev/null +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairCategoryModel.kt @@ -0,0 +1,11 @@ +package co.nilin.opex.matching.gateway.ports.postgres.model + +import org.springframework.data.annotation.Id +import org.springframework.data.relational.core.mapping.Table + +@Table("pair_category") +data class PairCategoryModel( + @Id val id: Long? = null, + val pair: String, + val category: PairCategory +) \ No newline at end of file diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairSettingModel.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairSettingModel.kt index f62832c5d..f36bdf34b 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairSettingModel.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/model/PairSettingModel.kt @@ -1,6 +1,7 @@ package co.nilin.opex.matching.gateway.ports.postgres.model import org.springframework.data.annotation.Id +import org.springframework.data.annotation.Transient import org.springframework.data.relational.core.mapping.Table import java.math.BigDecimal import java.time.LocalDateTime @@ -10,8 +11,13 @@ data class PairSettingModel( @Id val pair: String, var isAvailable: Boolean, - var minOrder : BigDecimal, - var maxOrder : BigDecimal, - var orderTypes : String, + var minOrder: BigDecimal, + var maxOrder: BigDecimal, + var orderTypes: String, var updateDate: LocalDateTime? = null, -) \ No newline at end of file + var internalChart: Boolean, + var globalChart: Boolean +) { + @Transient + var categories: List = emptyList() +} \ No newline at end of file diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/service/PairSettingInitializer.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/service/PairSettingInitializer.kt index 8bafa2161..da9bf47b0 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/service/PairSettingInitializer.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/service/PairSettingInitializer.kt @@ -46,7 +46,9 @@ class PairSettingInitializer( false, BigDecimal.ONE, BigDecimal.ONE, - "LIMIT_ORDER,MARKET_ORDER" + "LIMIT_ORDER,MARKET_ORDER", + true, + true ).then(pairSettingRepository.findByPair(pair)).awaitFirstOrNull() .also { if (it == null) logger.warn("Failed to insert pair: $pair") } ?: return@forEach diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/util/Convertor.kt b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/util/Convertor.kt index c039102d7..8737575f4 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/util/Convertor.kt +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/matching/gateway/ports/postgres/util/Convertor.kt @@ -12,6 +12,9 @@ fun PairSettingModel.toPairSetting(): PairSetting { maxOrder, orderTypes, updateDate, + internalChart, + globalChart, + categories ) } diff --git a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/resources/schema.sql b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/resources/schema.sql index c8a595b04..eb9af6fa8 100644 --- a/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/resources/schema.sql +++ b/matching-gateway/matching-gateway-port/matching-gateway-persister-postgres/src/main/resources/schema.sql @@ -23,5 +23,24 @@ $$ WHERE table_name = 'pair_setting' AND column_name = 'order_types') THEN ALTER TABLE pair_setting ADD COLUMN order_types varchar(255) NOT NULL default 'LIMIT_ORDER, MARKET_ORDER' ; END IF; + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pair_setting' + AND column_name = 'internal_chart') THEN ALTER TABLE pair_setting + ADD COLUMN internal_chart BOOLEAN NOT NULL default true; + END IF; + IF NOT EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pair_setting' AND column_name = 'global_chart') THEN ALTER TABLE pair_setting + ADD COLUMN global_chart BOOLEAN NOT NULL default true; + END IF; END -$$; \ No newline at end of file +$$; + +CREATE TABLE IF NOT EXISTS pair_category +( + id SERIAL PRIMARY KEY, + pair VARCHAR(72) NOT NULL REFERENCES pair_setting (pair), + category VARCHAR(255) NOT NULL, + UNIQUE (pair, category) +); From 86ab7833fff9ee662b944c861055d1ecf77fb9e6 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:26:49 +0330 Subject: [PATCH 12/22] Optimize market and accountant * Optimize market and accountant * Update tests based on optimizations --- .../opex/accountant/core/inout/RichOrder.kt | 4 +- .../accountant/core/inout/RichOrderUpdate.kt | 4 +- .../core/service/OrderManagerImpl.kt | 10 +- .../listener/config/AccountantKafkaConfig.kt | 112 +++++++-------- .../nilin/opex/api/core/inout/PairCategory.kt | 7 - .../opex/api/core/inout/PairInfoResponse.kt | 3 - .../nilin/opex/api/core/inout/PairSetting.kt | 3 - .../ports/opex/controller/MarketController.kt | 41 +----- .../opex/market/core/event/RichOrderUpdate.kt | 4 +- .../listener/config/KafkaConsumerConfig.kt | 17 ++- .../ports/postgres/dao/OrderRepository.kt | 108 +++++++++----- .../postgres/dao/OrderStatusRepository.kt | 6 +- .../ports/postgres/dao/TradeRepository.kt | 12 +- .../ports/postgres/impl/OrderPersisterImpl.kt | 26 +++- .../ports/postgres/impl/TradePersisterImpl.kt | 133 ++++++++++++++---- .../src/main/resources/schema.sql | 26 +++- .../ports/postgres/impl/OrderPersisterTest.kt | 42 ++++++ .../ports/postgres/impl/TradePersisterTest.kt | 47 +++++++ 18 files changed, 414 insertions(+), 191 deletions(-) delete mode 100644 api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrder.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrder.kt index 85f13c547..a09ebced1 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrder.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrder.kt @@ -4,6 +4,7 @@ import co.nilin.opex.matching.engine.core.model.MatchConstraint import co.nilin.opex.matching.engine.core.model.OrderDirection import co.nilin.opex.matching.engine.core.model.OrderType import java.math.BigDecimal +import java.time.LocalDateTime data class RichOrder( val orderId: Long? = 0, @@ -23,5 +24,6 @@ data class RichOrder( val quoteQuantity: BigDecimal, val executedQuantity: BigDecimal, val accumulativeQuoteQty: BigDecimal, - val status: Int = 0 + val status: Int = 0, + val createDate: LocalDateTime? ) : RichOrderEvent diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrderUpdate.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrderUpdate.kt index 81f515e26..78588f0d1 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrderUpdate.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/inout/RichOrderUpdate.kt @@ -1,13 +1,15 @@ package co.nilin.opex.accountant.core.inout import java.math.BigDecimal +import java.time.LocalDateTime data class RichOrderUpdate( val ouid: String, val price: BigDecimal, val quantity: BigDecimal, val remainedQuantity: BigDecimal, - val status: OrderStatus = OrderStatus.NEW + val status: OrderStatus = OrderStatus.NEW, + val updateDate: LocalDateTime?= LocalDateTime.now() ) : RichOrderEvent { fun executedQuantity(): BigDecimal = quantity.minus(remainedQuantity) diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/OrderManagerImpl.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/OrderManagerImpl.kt index 34bf51f4b..942157f83 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/OrderManagerImpl.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/OrderManagerImpl.kt @@ -212,12 +212,13 @@ open class OrderManagerImpl( richOrderPublisher.publish( RichOrderUpdate( order.ouid, - order.price.toBigDecimal(), - order.quantity.toBigDecimal(), - cancelOrderEvent.remainedQuantity.toBigDecimal(), + order.price.toBigDecimal().multiply(order.rightSideFraction), + order.origQuantity, + cancelOrderEvent.remainedQuantity.toBigDecimal().multiply(order.leftSideFraction), OrderStatus.CANCELED ) ) + return financialActionPersister.persist(listOf(financialAction)) /*publishFinancialAction(financialAction) return fa*/ @@ -253,7 +254,8 @@ open class OrderManagerImpl( OrderStatus.NEW.code } else { OrderStatus.PARTIALLY_FILLED.code - } + }, + LocalDateTime.now() ) ) } diff --git a/accountant/accountant-ports/accountant-eventlistener-kafka/src/main/kotlin/co/nilin/opex/accountant/ports/kafka/listener/config/AccountantKafkaConfig.kt b/accountant/accountant-ports/accountant-eventlistener-kafka/src/main/kotlin/co/nilin/opex/accountant/ports/kafka/listener/config/AccountantKafkaConfig.kt index 6ae6f37dd..48d81d1f0 100644 --- a/accountant/accountant-ports/accountant-eventlistener-kafka/src/main/kotlin/co/nilin/opex/accountant/ports/kafka/listener/config/AccountantKafkaConfig.kt +++ b/accountant/accountant-ports/accountant-eventlistener-kafka/src/main/kotlin/co/nilin/opex/accountant/ports/kafka/listener/config/AccountantKafkaConfig.kt @@ -1,4 +1,4 @@ -package co.nilin.opex.accountant.ports.kafka.listener.config +package co.nilin.opex.accountant.ports.kafka.listener.config import co.nilin.opex.accountant.core.inout.KycLevelUpdatedEvent import co.nilin.opex.accountant.ports.kafka.listener.consumer.* @@ -9,7 +9,6 @@ import co.nilin.opex.matching.engine.core.eventh.events.CoreEvent import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.serialization.StringDeserializer -import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnBean @@ -61,85 +60,83 @@ class AccountantKafkaConfig { fun withdrawRequestConsumerFactory(@Qualifier("consumerConfig") consumerConfigs: Map): ConsumerFactory { return DefaultKafkaConsumerFactory(consumerConfigs) } + @Bean("depositConsumerFactory") fun depositConsumerFactory(@Qualifier("consumerConfig") consumerConfigs: Map): ConsumerFactory { return DefaultKafkaConsumerFactory(consumerConfigs) } - @Autowired + @Bean("tradeKafkaListenerContainer") @ConditionalOnBean(TradeKafkaListener::class) - fun configureTradeListener( + fun tradeListenerContainer( tradeListener: TradeKafkaListener, @Qualifier("accountantEventKafkaTemplate") template: KafkaTemplate, @Qualifier("accountantConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("trades_.*")) containerProps.messageListener = tradeListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("TradeKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "trades.DLT") - container.start() + return container } - @Autowired + @Bean("eventKafkaListenerContainer") @ConditionalOnBean(EventKafkaListener::class) - fun configureEventListener( + fun eventListenerContainer( eventListener: EventKafkaListener, @Qualifier("accountantEventKafkaTemplate") template: KafkaTemplate, @Qualifier("accountantConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("events_.*")) containerProps.messageListener = eventListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("EventKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "events.DLT") - container.start() + return container } - @Autowired + @Bean("orderKafkaListenerContainer") @ConditionalOnBean(OrderKafkaListener::class) - fun configureOrderListener( + fun orderListenerContainer( orderListener: OrderKafkaListener, @Qualifier("accountantEventKafkaTemplate") template: KafkaTemplate, @Qualifier("accountantConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("orders_.*")) containerProps.messageListener = orderListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("OrderKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "orders.DLT") - container.start() + return container } - @Autowired + @Bean("tempEventKafkaListenerContainer") @ConditionalOnBean(TempEventKafkaListener::class) - fun configureTempEventListener( + fun tempEventListenerContainer( eventListener: TempEventKafkaListener, @Qualifier("accountantEventKafkaTemplate") template: KafkaTemplate, @Qualifier("accountantConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("tempevents")) containerProps.messageListener = eventListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("TempEventKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "tempevents.DLT") - container.start() + return container } - @Autowired + @Bean("faResponseKafkaListenerContainer") @ConditionalOnBean(FAResponseKafkaListener::class) - fun configureEventListener( + fun faResponseListenerContainer( eventListener: FAResponseKafkaListener, - //@Qualifier("accountantEventKafkaTemplate") template: KafkaTemplate, @Qualifier("faResponseConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("fiAction_response")) containerProps.messageListener = eventListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("FAResponseKafkaListenerContainer") - //TODO add error handler - //container.commonErrorHandler = createConsumerErrorHandler(template, "events.DLT") - container.start() + return container } @Bean("kycLevelUpdatedProducerFactory") @@ -152,69 +149,69 @@ class AccountantKafkaConfig { return KafkaTemplate(producerFactory) } - @Bean("withdrawRequestProducerFactory") - fun withdrawRequestProducerFactory(@Qualifier("consumerConfig") producerConfigs: Map): ProducerFactory { - return DefaultKafkaProducerFactory(producerConfigs) - } - - @Bean("withdrawRequestKafkaTemplate") - fun withdrawRequestKafkaTemplate(@Qualifier("withdrawRequestProducerFactory") producerFactory: ProducerFactory): KafkaTemplate { - return KafkaTemplate(producerFactory) - } - - @Bean("depositProducerFactory") - fun depositProducerFactory(@Qualifier("consumerConfig") producerConfigs: Map): ProducerFactory { - return DefaultKafkaProducerFactory(producerConfigs) - } - - @Bean("depositKafkaTemplate") - fun depositKafkaTemplate(@Qualifier("depositProducerFactory") producerFactory: ProducerFactory): KafkaTemplate { - return KafkaTemplate(producerFactory) - } - - @Autowired + @Bean("kycLevelUpdatedKafkaListenerContainer") @ConditionalOnBean(KycLevelUpdatedKafkaListener::class) - fun configureKycLevelUpdatedListener( + fun kycListenerContainer( listener: KycLevelUpdatedKafkaListener, @Qualifier("kycLevelUpdatedKafkaTemplate") template: KafkaTemplate, @Qualifier("KycConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("kyc_level_updated")) containerProps.messageListener = listener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("KycLevelUpdatedKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "kyc_level_updated.DLT") - container.start() + return container } - @Autowired + @Bean("withdrawRequestProducerFactory") + fun withdrawRequestProducerFactory(@Qualifier("consumerConfig") producerConfigs: Map): ProducerFactory { + return DefaultKafkaProducerFactory(producerConfigs) + } + + @Bean("withdrawRequestKafkaTemplate") + fun withdrawRequestKafkaTemplate(@Qualifier("withdrawRequestProducerFactory") producerFactory: ProducerFactory): KafkaTemplate { + return KafkaTemplate(producerFactory) + } + + @Bean("withdrawRequestKafkaListenerContainer") @ConditionalOnBean(WithdrawRequestKafkaListener::class) - fun configureWithdrawRequestEventListener( + fun withdrawRequestListenerContainer( listener: WithdrawRequestKafkaListener, @Qualifier("withdrawRequestKafkaTemplate") template: KafkaTemplate, @Qualifier("withdrawRequestConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("withdraw_request")) containerProps.messageListener = listener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("WithdrawRequestKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "withdraw_request.DLT") - container.start() + return container } - @Autowired + @Bean("depositProducerFactory") + fun depositProducerFactory(@Qualifier("consumerConfig") producerConfigs: Map): ProducerFactory { + return DefaultKafkaProducerFactory(producerConfigs) + } + + @Bean("depositKafkaTemplate") + fun depositKafkaTemplate(@Qualifier("depositProducerFactory") producerFactory: ProducerFactory): KafkaTemplate { + return KafkaTemplate(producerFactory) + } + + @Bean("depositKafkaListenerContainer") @ConditionalOnBean(DepositKafkaListener::class) - fun configureDepositRequestEventListener( + fun depositListenerContainer( listener: DepositKafkaListener, @Qualifier("depositKafkaTemplate") template: KafkaTemplate, @Qualifier("depositConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("deposit")) containerProps.messageListener = listener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("DepositKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "deposit.DLT") - container.start() + return container } private fun createConsumerErrorHandler(kafkaTemplate: KafkaTemplate<*, *>, dltTopic: String): CommonErrorHandler { @@ -224,5 +221,4 @@ class AccountantKafkaConfig { } return DefaultErrorHandler(recoverer, FixedBackOff(5_000, 20)) } - } \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt deleted file mode 100644 index b6b92ce16..000000000 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt +++ /dev/null @@ -1,7 +0,0 @@ -package co.nilin.opex.api.core.inout - -enum class PairCategory { - REAL_ASSET_TOKEN, - FIAT, - CRYPTO -} \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt index 1fed02103..d6c00fe0e 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt @@ -10,7 +10,4 @@ data class PairInfoResponse( val minOrder : BigDecimal, val maxOrder : BigDecimal, val orderTypes : String, - val internalChart: Boolean, - val globalChart: Boolean, - val categories: List = emptyList() ) \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt index 782a09a0e..b3d7d5946 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt @@ -10,7 +10,4 @@ class PairSetting( val maxOrder : BigDecimal, val orderTypes : String, val updateDate: LocalDateTime? = null, - val internalChart: Boolean, - val globalChart: Boolean, - val categories: List = emptyList() ) \ No newline at end of file diff --git a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt index 6ce743d62..290c55827 100644 --- a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt +++ b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt @@ -18,7 +18,11 @@ import io.swagger.v3.oas.annotations.tags.Tag import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.springframework.beans.factory.annotation.Value -import org.springframework.web.bind.annotation.* +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController import java.math.BigDecimal import java.time.ZoneId @@ -100,45 +104,12 @@ Response body: isAvailable = isAvailable, minOrder = minOrder, maxOrder = maxOrder, - orderTypes = orderTypes, - internalChart = internalChart, - globalChart = globalChart, - categories = categories + orderTypes = orderTypes ) } } } - @GetMapping("/pair/categories") - @Operation( - summary = "Get trading pair categories", - description = """ -Security: -- Public endpoint. No Bearer token is required. - -Source of values: -- Category names returned here can be used by clients to filter pairs. - -Response body: -- Array of pair category names. - """, - responses = [ - ApiResponse( - responseCode = "200", - description = "Trading pair categories returned successfully.", - content = [ - Content( - mediaType = "application/json", - array = ArraySchema(schema = Schema(type = "string")) - ) - ] - ) - ] - ) - fun getPairCategories(): List { - return PairCategory.entries.map { it.name } - } - @GetMapping("/chain") @Operation( summary = "Get chains", diff --git a/market/market-core/src/main/kotlin/co/nilin/opex/market/core/event/RichOrderUpdate.kt b/market/market-core/src/main/kotlin/co/nilin/opex/market/core/event/RichOrderUpdate.kt index df048861e..98ce274fe 100644 --- a/market/market-core/src/main/kotlin/co/nilin/opex/market/core/event/RichOrderUpdate.kt +++ b/market/market-core/src/main/kotlin/co/nilin/opex/market/core/event/RichOrderUpdate.kt @@ -2,13 +2,15 @@ package co.nilin.opex.market.core.event import co.nilin.opex.market.core.inout.OrderStatus import java.math.BigDecimal +import java.time.LocalDateTime data class RichOrderUpdate( val ouid: String, val price: BigDecimal, val quantity: BigDecimal, val remainedQuantity: BigDecimal, - val status: OrderStatus = OrderStatus.NEW + val status: OrderStatus = OrderStatus.NEW, + val updateDate: LocalDateTime? = LocalDateTime.now() ) : RichOrderEvent { fun executedQuantity(): BigDecimal = quantity.minus(remainedQuantity) diff --git a/market/market-ports/market-eventlistener-kafka/src/main/kotlin/co/nilin/opex/market/ports/kafka/listener/config/KafkaConsumerConfig.kt b/market/market-ports/market-eventlistener-kafka/src/main/kotlin/co/nilin/opex/market/ports/kafka/listener/config/KafkaConsumerConfig.kt index 7a55d05a4..854c3fa7b 100644 --- a/market/market-ports/market-eventlistener-kafka/src/main/kotlin/co/nilin/opex/market/ports/kafka/listener/config/KafkaConsumerConfig.kt +++ b/market/market-ports/market-eventlistener-kafka/src/main/kotlin/co/nilin/opex/market/ports/kafka/listener/config/KafkaConsumerConfig.kt @@ -7,7 +7,6 @@ import co.nilin.opex.market.ports.kafka.listener.consumer.TradeKafkaListener import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.serialization.StringDeserializer -import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnBean @@ -52,34 +51,34 @@ class KafkaConsumerConfig { return DefaultKafkaConsumerFactory(consumerConfigs) } - @Autowired + @Bean("marketTradeKafkaListenerContainer") @ConditionalOnBean(TradeKafkaListener::class) - fun configureTradeListener( + fun tradeListenerContainer( tradeListener: TradeKafkaListener, template: KafkaTemplate, @Qualifier("richTradeConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("richTrade")) containerProps.messageListener = tradeListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("marketTradeKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "richTrade.DLT") - container.start() + return container } - @Autowired + @Bean("marketOrderKafkaListenerContainer") @ConditionalOnBean(OrderKafkaListener::class) - fun configureOrderListener( + fun orderListenerContainer( orderListener: OrderKafkaListener, template: KafkaTemplate, @Qualifier("richOrderConsumerFactory") consumerFactory: ConsumerFactory - ) { + ): ConcurrentMessageListenerContainer { val containerProps = ContainerProperties(Pattern.compile("richOrder")) containerProps.messageListener = orderListener val container = ConcurrentMessageListenerContainer(consumerFactory, containerProps) container.setBeanName("marketOrderKafkaListenerContainer") container.commonErrorHandler = createConsumerErrorHandler(template, "richOrder.DLT") - container.start() + return container } private fun createConsumerErrorHandler(kafkaTemplate: KafkaTemplate<*, *>, dltTopic: String): CommonErrorHandler { diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt index 6f7bbd9a3..cc394b310 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt @@ -41,6 +41,14 @@ interface OrderRepository : ReactiveCrudRepository { origClientOrderId: String, ): Mono + @Query("update orders set update_date = :updateDate where ouid = :ouid") + fun touchUpdateDateByOuid( + @Param("ouid") + ouid: String, + @Param("updateDate") + updateDate: LocalDateTime = LocalDateTime.now() + ): Mono + @Query( """ select * from orders @@ -138,35 +146,55 @@ interface OrderRepository : ReactiveCrudRepository { @Query( """ -select o.symbol, - o.ouid, - o.order_type, - o.side, - o.price, - o.quantity, - o.quote_quantity, - os.executed_quantity, - o.taker_fee, - o.maker_fee, - os.status as status_code, - os.appearance, - o.create_date, - os.date as update_date, - o.uuid -from orders o - left join (select * - from order_status os1 - where os1.date = (select max(os2.date) - from order_status os2 - where os2.ouid = os1.ouid)) os on o.ouid = os.ouid - WHERE (:uuid is null or o.uuid = :uuid) - and (:symbol is null or o.symbol = :symbol) - and (:startTime is null or o.create_date >= :startTime) - and (:endTime is null or o.create_date <= :endTime) - and (:orderType is null or o.order_type = :orderType) - and (:direction is null or o.side = :direction) -order by create_date desc - limit :limit offset :offset; +with filtered_orders as ( + select o.symbol, + o.ouid, + o.order_type, + o.side, + o.price, + o.quantity, + o.quote_quantity, + o.taker_fee, + o.maker_fee, + o.create_date, + o.uuid + from orders o + where (:uuid is null or o.uuid = :uuid) + and (:symbol is null or o.symbol = :symbol) + and (:startTime is null or o.create_date >= :startTime) + and (:endTime is null or o.create_date <= :endTime) + and (:orderType is null or o.order_type = :orderType) + and (:direction is null or o.side = :direction) + order by o.create_date desc + limit :limit offset :offset +) +select fo.symbol, + fo.ouid, + fo.order_type, + fo.side, + fo.price, + fo.quantity, + fo.quote_quantity, + os.executed_quantity, + fo.taker_fee, + fo.maker_fee, + os.status as status_code, + os.appearance, + fo.create_date, + os.date as update_date, + fo.uuid +from filtered_orders fo +left join lateral ( + select s.executed_quantity, + s.status, + s.appearance, + s.date + from order_status s + where s.ouid = fo.ouid + order by s.appearance desc, s.executed_quantity desc nulls last, s.date desc, s.id desc + limit 1 +) os on true +order by fo.create_date desc; """ ) fun findByCriteria( @@ -228,14 +256,20 @@ SELECT FROM orders o LEFT JOIN ( - SELECT DISTINCT ON (ouid) - ouid, - executed_quantity, - status, - appearance, - date - FROM order_status - ORDER BY ouid, date DESC + SELECT ranked.ouid, + ranked.executed_quantity, + ranked.status, + ranked.appearance, + ranked.date + FROM ( + SELECT os.*, + ROW_NUMBER() OVER ( + PARTITION BY os.ouid + ORDER BY os.appearance DESC, os.executed_quantity DESC NULLS LAST, os.date DESC, os.id DESC + ) AS rnk + FROM order_status os + ) ranked + WHERE ranked.rnk = 1 ) os ON os.ouid = o.ouid diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderStatusRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderStatusRepository.kt index 72aa02548..b7632fc7b 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderStatusRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderStatusRepository.kt @@ -30,7 +30,11 @@ interface OrderStatusRepository : ReactiveCrudRepository @Query( """ WITH ranked_order_status AS ( - SELECT *, ROW_NUMBER() OVER (PARTITION BY ouid ORDER BY appearance DESC, executed_quantity DESC) AS rnk + SELECT *, + ROW_NUMBER() OVER ( + PARTITION BY ouid + ORDER BY appearance DESC, executed_quantity DESC NULLS LAST, date DESC, id DESC + ) AS rnk FROM order_status WHERE ouid = :ouid ) diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt index c0eacbe01..517db2d74 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt @@ -22,6 +22,14 @@ interface TradeRepository : ReactiveCrudRepository { @Query("select * from trades where symbol = :symbol order by create_date desc limit 1") fun findMostRecentBySymbol(symbol: String): Flux + @Query("select * from trades where symbol = :symbol and trade_id = :tradeId limit 1") + fun findBySymbolAndTradeId( + @Param("symbol") + symbol: String, + @Param("tradeId") + tradeId: Long + ): Mono + @Query("select * from trades where symbol = :symbol order by create_date desc limit :limit") fun findBySymbolSortDescendingByCreateDate( @Param("symbol") @@ -134,8 +142,8 @@ interface TradeRepository : ReactiveCrudRepository { CASE WHEN t.taker_uuid = :uuid - THEN (to2.side = 'ASK') - ELSE (mo.side = 'ASK') + THEN (to2.side = 'BID') + ELSE (mo.side = 'BID') END AS isBuyer, (t.maker_uuid = :uuid) AS isMaker diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt index c6870c2cf..87a7d7a3d 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt @@ -17,6 +17,8 @@ import kotlinx.coroutines.reactive.awaitFirstOrNull import kotlinx.coroutines.reactor.awaitSingle import kotlinx.coroutines.reactor.awaitSingleOrNull import org.slf4j.LoggerFactory +import org.springframework.dao.DataIntegrityViolationException +import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -55,7 +57,15 @@ class OrderPersisterImpl( LocalDateTime.now(), LocalDateTime.now() ) - orderRepository.save(orderModel).awaitFirstOrNull() + try { + orderRepository.save(orderModel).awaitFirstOrNull() + } catch (e: DuplicateKeyException) { + logger.info("order ${order.ouid} is duplicate; skipping create flow") + return + } catch (e: DataIntegrityViolationException) { + logger.info("order ${order.ouid} is duplicate; skipping create flow") + return + } logger.info("order ${order.ouid} saved") orderStatusRepository.insert( @@ -83,12 +93,18 @@ class OrderPersisterImpl( @Transactional override suspend fun update(orderUpdate: RichOrderUpdate) { + + val updateTime = orderUpdate.updateDate ?: LocalDateTime.now() + + orderRepository.touchUpdateDateByOuid(orderUpdate.ouid, updateTime).awaitFirstOrNull() + orderStatusRepository.insert( orderUpdate.ouid, orderUpdate.executedQuantity(), orderUpdate.accumulativeQuoteQuantity(), orderUpdate.status.code, - orderUpdate.status.orderOfAppearance + orderUpdate.status.orderOfAppearance, + updateTime ).awaitFirstOrNull() logger.info("OrderStatus ${orderUpdate.ouid} updated with status of ${orderUpdate.status}") @@ -101,7 +117,11 @@ class OrderPersisterImpl( openOrderRepository.delete(orderUpdate.ouid).awaitSingleOrNull() logger.info("Order ${orderUpdate.ouid} deleted from open orders") } - val order = orderRepository.findByOuid(orderUpdate.ouid).awaitFirstOrNull() ?: return + val order = orderRepository.findByOuid(orderUpdate.ouid).awaitFirstOrNull() + ?: run { + logger.info("Order ${orderUpdate.ouid} not found for update event, SKIPPED") + return + } marketOrderProducer.openOrderUpdate(order.uuid, order.symbol) } diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt index b229535a4..a257a1d99 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt @@ -9,11 +9,13 @@ import co.nilin.opex.market.ports.postgres.model.TradeModel import co.nilin.opex.market.ports.postgres.util.RedisCacheHelper import kotlinx.coroutines.reactive.awaitFirstOrNull import org.slf4j.LoggerFactory +import org.springframework.dao.DataIntegrityViolationException +import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Component -import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime import java.time.ZoneId import java.util.* +import java.util.concurrent.atomic.AtomicLong @Component class TradePersisterImpl( @@ -23,33 +25,42 @@ class TradePersisterImpl( private val logger = LoggerFactory.getLogger(TradePersisterImpl::class.java) - @Transactional override suspend fun save(trade: RichTrade) { val pair = trade.pair.split("_") + val tradeModel = TradeModel( + null, + trade.id, + trade.pair, + pair[0].uppercase(), + pair[1].uppercase(), + trade.matchedPrice, + trade.matchedQuantity, + trade.takerPrice, + trade.makerPrice, + trade.takerCommision, + trade.makerCommision, + trade.takerCommisionAsset, + trade.makerCommisionAsset, + trade.tradeDateTime, + trade.makerOuid, + trade.takerOuid, + trade.makerUuid, + trade.takerUuid, + LocalDateTime.now() + ) - val tradeEntity = tradeRepository.save( - TradeModel( - null, - trade.id, - trade.pair, - pair[0].uppercase(), - pair[1].uppercase(), - trade.matchedPrice, - trade.matchedQuantity, - trade.takerPrice, - trade.makerPrice, - trade.takerCommision, - trade.makerCommision, - trade.takerCommisionAsset, - trade.makerCommisionAsset, - trade.tradeDateTime, - trade.makerOuid, - trade.takerOuid, - trade.makerUuid, - trade.takerUuid, - LocalDateTime.now() - ) - ).awaitFirstOrNull() + val tradeEntity = try { + tradeRepository.save(tradeModel).awaitFirstOrNull() + } catch (e: DuplicateKeyException) { + ensureNotCollision(tradeModel, trade) + return + } catch (e: DataIntegrityViolationException) { + if (!isDuplicateTradeViolation(e)) { + throw e + } + ensureNotCollision(tradeModel, trade) + return + } logger.info("RichTrade ${trade.id} saved") //calculateTradeVolume(trade, pair[0].uppercase(), pair[1].uppercase()) // Moved to accountant updateCache(trade, tradeEntity) @@ -85,4 +96,76 @@ class TradePersisterImpl( logger.info("Could not update recentTrades cache") } } + + private suspend fun ensureNotCollision(incomingTrade: TradeModel, originalTrade: RichTrade) { + val existingTrade = tradeRepository.findBySymbolAndTradeId( + incomingTrade.symbol, + incomingTrade.tradeId + ).awaitFirstOrNull() ?: throw IllegalStateException( + "Duplicate trade conflict detected but existing row not found for symbol=${incomingTrade.symbol}, tradeId=${incomingTrade.tradeId}" + ) + + if (isSameTradePayload(existingTrade, incomingTrade)) { + logger.info("RichTrade ${incomingTrade.tradeId} for ${incomingTrade.symbol} is duplicate delivery; skipping") + return + } + + // Real ID collision (e.g. Redis counter reset): persist under a new synthetic ID and continue + logger.error( + "Trade ID collision for symbol=${incomingTrade.symbol}, tradeId=${incomingTrade.tradeId}. " + + "Saving colliding trade under a new synthetic ID." + ) + //todo cast the tradeId to BigInteger + val newId = generateUniqueId() + logger.info("The old tradeId ${incomingTrade.tradeId} - The new tradeId: $newId") + val reassigned = TradeModel( + null, + newId, + incomingTrade.symbol, + incomingTrade.baseAsset, + incomingTrade.quoteAsset, + incomingTrade.matchedPrice, + incomingTrade.matchedQuantity, + incomingTrade.takerPrice, + incomingTrade.makerPrice, + incomingTrade.takerCommission, + incomingTrade.makerCommission, + incomingTrade.takerCommissionAsset, + incomingTrade.makerCommissionAsset, + incomingTrade.tradeDate, + incomingTrade.makerOuid, + incomingTrade.takerOuid, + incomingTrade.makerUuid, + incomingTrade.takerUuid, + incomingTrade.createDate + ) + val saved = tradeRepository.save(reassigned).awaitFirstOrNull() + updateCache(originalTrade, saved) + } + + private fun isSameTradePayload(existing: TradeModel, incoming: TradeModel): Boolean { + return existing.makerOuid == incoming.makerOuid && + existing.takerOuid == incoming.takerOuid && + existing.matchedPrice.compareTo(incoming.matchedPrice) == 0 && + existing.matchedQuantity.compareTo(incoming.matchedQuantity) == 0 && + existing.tradeDate == incoming.tradeDate && + existing.makerCommission == incoming.makerCommission && + existing.takerCommission == incoming.takerCommission && + existing.makerCommissionAsset == incoming.makerCommissionAsset && + existing.takerCommissionAsset == incoming.takerCommissionAsset + } + + private fun isDuplicateTradeViolation(exception: Throwable): Boolean { + val errorText = buildString { + append(exception.message.orEmpty()) + append(' ') + append(exception.cause?.message.orEmpty()) + } + return errorText.contains("uq_trades_symbol_trade_id", ignoreCase = true) || + errorText.contains("duplicate key value", ignoreCase = true) + } + + private fun generateUniqueId(): Long { + return AtomicLong(System.currentTimeMillis() * 1000).incrementAndGet() + } } \ No newline at end of file diff --git a/market/market-ports/market-persister-postgres/src/main/resources/schema.sql b/market/market-ports/market-persister-postgres/src/main/resources/schema.sql index 37cb1ce0c..294115909 100644 --- a/market/market-ports/market-persister-postgres/src/main/resources/schema.sql +++ b/market/market-ports/market-persister-postgres/src/main/resources/schema.sql @@ -33,6 +33,8 @@ CREATE TABLE IF NOT EXISTS order_status date TIMESTAMP NOT NULL, UNIQUE (ouid, status, appearance, executed_quantity) ); +CREATE INDEX IF NOT EXISTS idx_order_status_ouid_rank + ON order_status (ouid, appearance DESC, executed_quantity DESC, date DESC, id DESC); CREATE TABLE IF NOT EXISTS open_orders ( @@ -42,6 +44,9 @@ CREATE TABLE IF NOT EXISTS open_orders status INTEGER NOT NULL ); +CREATE INDEX IF NOT EXISTS idx_orders_uuid_create_date ON orders (uuid, create_date DESC); +CREATE INDEX IF NOT EXISTS idx_orders_uuid_symbol_create_date ON orders (uuid, symbol, create_date DESC); + CREATE TABLE IF NOT EXISTS trades ( id SERIAL PRIMARY KEY, @@ -67,6 +72,26 @@ CREATE TABLE IF NOT EXISTS trades CREATE INDEX IF NOT EXISTS idx_trades_symbol on trades (symbol); CREATE INDEX IF NOT EXISTS idx_trades_create_date on trades (create_date); +ALTER TABLE trades + ALTER COLUMN id TYPE BIGINT, + ALTER COLUMN trade_id TYPE BIGINT; +ALTER SEQUENCE trades_id_seq AS BIGINT; + +WITH duplicate_trades AS ( + SELECT id + FROM ( + SELECT id, + ROW_NUMBER() OVER (PARTITION BY symbol, trade_id ORDER BY id) AS rn + FROM trades + ) ranked + WHERE rn > 1 +) +DELETE +FROM trades t + USING duplicate_trades d +WHERE t.id = d.id; +CREATE UNIQUE INDEX IF NOT EXISTS uq_trades_symbol_trade_id on trades (symbol, trade_id); + CREATE OR REPLACE FUNCTION interval_generator( start_ts TIMESTAMP without TIME ZONE, end_ts TIMESTAMP without TIME ZONE, @@ -85,4 +110,3 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; - diff --git a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt index 0a533641a..ba1077369 100644 --- a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt +++ b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt @@ -9,9 +9,12 @@ import co.nilin.opex.market.ports.postgres.util.RedisCacheHelper import io.mockk.coEvery import io.mockk.every import io.mockk.mockk +import io.mockk.verify import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThatNoException import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.springframework.dao.DuplicateKeyException import reactor.core.publisher.Mono class OrderPersisterTest { @@ -57,6 +60,9 @@ class OrderPersisterTest { @Test fun givenOrderRepo_whenUpdateRichOrder_thenSuccess(): Unit = runBlocking { + every { + orderRepository.touchUpdateDateByOuid(any(), any()) + } returns Mono.empty() every { orderStatusRepository.insert(any(), any(), any(), any(), any(), any()) } returns Mono.empty() @@ -76,4 +82,40 @@ class OrderPersisterTest { assertThatNoException().isThrownBy { runBlocking { orderPersister.update(VALID.RICH_ORDER_UPDATE) } } } + + @Test + fun givenDuplicateOrderCreate_whenSaveRichOrder_thenIgnoredAsIdempotent(): Unit = runBlocking { + every { + orderRepository.save(any()) + } returns Mono.error(DuplicateKeyException("duplicate order")) + + assertThatNoException().isThrownBy { runBlocking { orderPersister.save(VALID.RICH_ORDER) } } + + verify(exactly = 0) { + orderStatusRepository.insert(any(), any(), any(), any(), any(), any()) + } + } + + //To have race condition between RichOrder and UpdateRichOrder,we will temporarily skip this test + +// @Test +// fun givenMissingOrder_whenUpdateRichOrder_thenFailBeforeSideEffects(): Unit = runBlocking { +// every { +// orderRepository.findByOuid(any()) +// } returns Mono.empty() +// +// assertThrows { +// runBlocking { orderPersister.update(VALID.RICH_ORDER_UPDATE) } +// } +// +// verify(exactly = 0) { +// orderRepository.touchUpdateDateByOuid(any(), any()) +// } +// verify(exactly = 0) { +// orderStatusRepository.insert(any(), any(), any(), any(), any(), any()) +// } +// verify(exactly = 0) { +// openOrderRepository.insertOrUpdate(any(), any(), any()) +// } +// } } diff --git a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterTest.kt b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterTest.kt index 2a2f8f531..308f9f9cc 100644 --- a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterTest.kt +++ b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterTest.kt @@ -2,13 +2,18 @@ package co.nilin.opex.market.ports.postgres.impl import co.nilin.opex.market.ports.postgres.dao.TradeRepository import co.nilin.opex.market.ports.postgres.impl.sample.VALID +import co.nilin.opex.market.ports.postgres.model.TradeModel import co.nilin.opex.market.ports.postgres.util.RedisCacheHelper import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThatNoException import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.springframework.dao.DataIntegrityViolationException +import org.springframework.dao.DuplicateKeyException import reactor.core.publisher.Mono +import java.math.BigDecimal class TradePersisterTest { @@ -22,4 +27,46 @@ class TradePersisterTest { every { tradeRepository.save(any()) } returns Mono.just(VALID.TRADE_MODEL) assertThatNoException().isThrownBy { runBlocking { tradePersister.save(VALID.RICH_TRADE) } } } + + @Test + fun givenDuplicateTrade_whenSaveRichTrade_thenIgnoredAsIdempotent(): Unit = runBlocking { + every { tradeRepository.save(any()) } returnsMany listOf( + Mono.error(DuplicateKeyException("Duplicate key")), + Mono.just(VALID.TRADE_MODEL) + ) + every { tradeRepository.findBySymbolAndTradeId(any(), any()) } returns Mono.just(VALID.TRADE_MODEL) + assertThatNoException().isThrownBy { runBlocking { tradePersister.save(VALID.RICH_TRADE) } } + } + + @Test + fun givenTradeIdCollision_whenSaveRichTrade_thenThrow() { + every { tradeRepository.save(any()) } returns Mono.error(DuplicateKeyException("duplicate trade")) + every { tradeRepository.findBySymbolAndTradeId(any(), any()) } returns Mono.just( + TradeModel( + VALID.TRADE_MODEL.id, + VALID.TRADE_MODEL.tradeId, + VALID.TRADE_MODEL.symbol, + VALID.TRADE_MODEL.baseAsset, + VALID.TRADE_MODEL.quoteAsset, + VALID.TRADE_MODEL.matchedPrice, + VALID.TRADE_MODEL.matchedQuantity.add(BigDecimal.ONE), + VALID.TRADE_MODEL.takerPrice, + VALID.TRADE_MODEL.makerPrice, + VALID.TRADE_MODEL.takerCommission, + VALID.TRADE_MODEL.makerCommission, + VALID.TRADE_MODEL.takerCommissionAsset, + VALID.TRADE_MODEL.makerCommissionAsset, + VALID.TRADE_MODEL.tradeDate, + VALID.TRADE_MODEL.makerOuid, + VALID.TRADE_MODEL.takerOuid, + VALID.TRADE_MODEL.makerUuid, + VALID.TRADE_MODEL.takerUuid, + VALID.TRADE_MODEL.createDate + ) + ) + + assertThrows { + runBlocking { tradePersister.save(VALID.RICH_TRADE) } + } + } } From 6331770a182f53341fbe61983e60c924b641b114 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:09:00 +0330 Subject: [PATCH 13/22] Optimize wallet and api --- .../src/main/resources/application.yml | 1 + .../walletproxy/config/WebClientConfig.kt | 39 ++++++++++++++++--- .../opex/api/app/config/RateLimitConfig.kt | 13 +++++-- .../src/main/resources/application.yml | 9 +++-- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/accountant/accountant-app/src/main/resources/application.yml b/accountant/accountant-app/src/main/resources/application.yml index 1f08d8860..f191ca34c 100644 --- a/accountant/accountant-app/src/main/resources/application.yml +++ b/accountant/accountant-app/src/main/resources/application.yml @@ -47,6 +47,7 @@ spring: instance-id: ${spring.application.name}:${server.port} healthCheckInterval: 20s prefer-ip-address: true + query-passing: true config: import: vault://secret/${spring.application.name} management: diff --git a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/config/WebClientConfig.kt b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/config/WebClientConfig.kt index 2ba1c3c02..2f9bbe64c 100644 --- a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/config/WebClientConfig.kt +++ b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/config/WebClientConfig.kt @@ -1,25 +1,52 @@ package co.nilin.opex.accountant.ports.walletproxy.config +import io.netty.channel.ChannelOption import org.springframework.cloud.client.ServiceInstance import org.springframework.cloud.client.loadbalancer.reactive.ReactiveLoadBalancer import org.springframework.cloud.client.loadbalancer.reactive.ReactorLoadBalancerExchangeFilterFunction import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.http.client.reactive.ReactorClientHttpConnector import org.springframework.web.reactive.function.client.WebClient import org.zalando.logbook.Logbook import org.zalando.logbook.netty.LogbookClientHandler import reactor.netty.http.client.HttpClient +import reactor.netty.resources.ConnectionProvider +import java.time.Duration @Configuration class WebClientConfig { @Bean - fun webClient(loadBalancerFactory: ReactiveLoadBalancer.Factory, logbook: Logbook): WebClient { - val client = HttpClient.create().doOnConnected { it.addHandlerLast(LogbookClientHandler(logbook)) } + fun webClient( + loadBalancerFactory: ReactiveLoadBalancer.Factory, + logbook: Logbook + ): WebClient { + + val connectionProvider = ConnectionProvider.builder("accountant-wallet") + .maxIdleTime(Duration.ofSeconds(20)) + .maxLifeTime(Duration.ofMinutes(5)) + .pendingAcquireTimeout(Duration.ofSeconds(5)) + .evictInBackground(Duration.ofSeconds(30)) + .lifo() + .build() + + val client = HttpClient.create(connectionProvider) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 3000) + .responseTimeout(Duration.ofSeconds(10)) + .keepAlive(true) + .doOnConnected { + it.addHandlerLast(LogbookClientHandler(logbook)) + } + return WebClient.builder() - //.clientConnector(ReactorClientHttpConnector(client)) - .filter(ReactorLoadBalancerExchangeFilterFunction(loadBalancerFactory, emptyList())) + .clientConnector(ReactorClientHttpConnector(client)) + .filter( + ReactorLoadBalancerExchangeFilterFunction( + loadBalancerFactory, + emptyList() + ) + ) .build() } - -} +} \ No newline at end of file diff --git a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/RateLimitConfig.kt b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/RateLimitConfig.kt index fd9167451..054d639d8 100644 --- a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/RateLimitConfig.kt +++ b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/RateLimitConfig.kt @@ -53,12 +53,17 @@ class RateLimitConfig( return ReactiveSecurityContextHolder.getContext() .mapNotNull { it.authentication } .filter { it.isAuthenticated } - .flatMap { auth -> - applyRateLimit(auth.name, exchange, chain, groupId) + .map { auth -> + Mono.defer { + applyRateLimit(auth.name, exchange, chain, groupId) + } } - .switchIfEmpty( - chain.filter(exchange) + .defaultIfEmpty( + Mono.defer { + chain.filter(exchange) + } ) + .flatMap { it } } private fun applyRateLimit( diff --git a/wallet/wallet-app/src/main/resources/application.yml b/wallet/wallet-app/src/main/resources/application.yml index 488d21278..ec67e2e77 100644 --- a/wallet/wallet-app/src/main/resources/application.yml +++ b/wallet/wallet-app/src/main/resources/application.yml @@ -8,6 +8,8 @@ management: endpoint: health: show-details: when_authorized + probes: + enabled: true metrics: enabled: true prometheus: @@ -66,9 +68,10 @@ spring: host: ${CONSUL_HOST:localhost} port: 8500 discovery: - #healthCheckPath: ${management.context-path}/health instance-id: ${spring.application.name}:${server.port} - healthCheckInterval: 20s + health-check-path: /actuator/health/liveness + health-check-interval: 10s + health-check-timeout: 5s prefer-ip-address: true config: import: vault://secret/${spring.application.name} @@ -174,6 +177,4 @@ logging: co.nilin: INFO reactor.netty.http.client: INFO org.zalando.logbook: TRACE - org.hibernate.SQL: DEBUG - logging.level.org.hibernate.type.descriptor.sql.BasicBinder: TRACE swagger.authUrl: ${SWAGGER_AUTH_URL:https://api.opex.dev/auth}/realms/opex/protocol/openid-connect/token From 05b309a41c84138797abf2b56e49ecb47ff43921 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:09:39 +0200 Subject: [PATCH 14/22] Chore/optimize api wallet --- .../opex/api/app/config/WebClientConfig.kt | 71 ++++++++++++++----- .../src/main/resources/application.yml | 34 +++++++-- .../ports/proxy/config/ProxyDispatchers.kt | 28 +++++++- .../proxy/impl/MarketUserDataProxyImpl.kt | 7 +- .../proxy/impl/MatchingGatewayProxyImpl.kt | 8 ++- device-management/pom.xml | 2 +- pom.xml | 2 +- 7 files changed, 123 insertions(+), 29 deletions(-) diff --git a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/WebClientConfig.kt b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/WebClientConfig.kt index a89032080..c2d67d527 100644 --- a/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/WebClientConfig.kt +++ b/api/api-app/src/main/kotlin/co/nilin/opex/api/app/config/WebClientConfig.kt @@ -26,22 +26,58 @@ class WebClientConfig( private val logbook: Logbook, @Value("\${app.auth.url}") private val url: String, + @Value("\${app.http.client.wiretap.enabled:false}") + private val wiretapEnabled: Boolean, + @Value("\${app.http.client.general.max-connections:300}") + private val generalMaxConnections: Int, + @Value("\${app.http.client.general.pending-acquire-max-count:1000}") + private val generalPendingAcquireMaxCount: Int, + @Value("\${app.http.client.general.max-idle-seconds:30}") + private val generalMaxIdleSeconds: Long, + @Value("\${app.http.client.general.max-life-seconds:120}") + private val generalMaxLifeSeconds: Long, + @Value("\${app.http.client.general.pending-acquire-timeout-seconds:30}") + private val generalPendingAcquireTimeoutSeconds: Long, + @Value("\${app.http.client.general.connect-timeout-millis:5000}") + private val generalConnectTimeoutMillis: Int, + @Value("\${app.http.client.general.response-timeout-seconds:30}") + private val generalResponseTimeoutSeconds: Long, + @Value("\${app.http.client.keycloak.max-connections:150}") + private val keycloakMaxConnections: Int, + @Value("\${app.http.client.keycloak.pending-acquire-max-count:500}") + private val keycloakPendingAcquireMaxCount: Int, + @Value("\${app.http.client.keycloak.max-idle-seconds:30}") + private val keycloakMaxIdleSeconds: Long, + @Value("\${app.http.client.keycloak.max-life-seconds:120}") + private val keycloakMaxLifeSeconds: Long, + @Value("\${app.http.client.keycloak.pending-acquire-timeout-seconds:60}") + private val keycloakPendingAcquireTimeoutSeconds: Long, + @Value("\${app.http.client.keycloak.connect-timeout-millis:10000}") + private val keycloakConnectTimeoutMillis: Int, + @Value("\${app.http.client.keycloak.response-timeout-seconds:10}") + private val keycloakResponseTimeoutSeconds: Long, ) { private val provider = ConnectionProvider.builder("apiPool") - .maxConnections(150) - .pendingAcquireMaxCount(100) - .maxIdleTime(Duration.ofSeconds(30)) - .maxLifeTime(Duration.ofMinutes(2)) - .pendingAcquireTimeout(Duration.ofSeconds(10)) + .maxConnections(generalMaxConnections) + .pendingAcquireMaxCount(generalPendingAcquireMaxCount) + .maxIdleTime(Duration.ofSeconds(generalMaxIdleSeconds)) + .maxLifeTime(Duration.ofSeconds(generalMaxLifeSeconds)) + .pendingAcquireTimeout(Duration.ofSeconds(generalPendingAcquireTimeoutSeconds)) .evictInBackground(Duration.ofMinutes(1)) .build() - private val client = HttpClient.create(provider) - .wiretap("reactor.netty.http.client.HttpClient", LogLevel.DEBUG, AdvancedByteBufFormat.SIMPLE) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000) - .responseTimeout(Duration.ofSeconds(30)) - .keepAlive(true) - .doOnConnected { it.addHandlerLast(LogbookClientHandler(logbook)) } + private val client = HttpClient.create(provider).let { + val configured = if (wiretapEnabled) { + it.wiretap("reactor.netty.http.client.HttpClient", LogLevel.DEBUG, AdvancedByteBufFormat.SIMPLE) + } else { + it + } + configured + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, generalConnectTimeoutMillis) + .responseTimeout(Duration.ofSeconds(generalResponseTimeoutSeconds)) + .keepAlive(true) + .doOnConnected { conn -> conn.addHandlerLast(LogbookClientHandler(logbook)) } + } @Bean("generalWebClient") @@ -63,16 +99,17 @@ class WebClientConfig( @Bean("keycloakWebClient") fun keycloakWebClient(logbook: Logbook): WebClient { val provider = ConnectionProvider.builder("keycloakPool") - .maxConnections(100) - .maxIdleTime(Duration.ofSeconds(30)) - .maxLifeTime(Duration.ofMinutes(2)) - .pendingAcquireTimeout(Duration.ofSeconds(60)) + .maxConnections(keycloakMaxConnections) + .pendingAcquireMaxCount(keycloakPendingAcquireMaxCount) + .maxIdleTime(Duration.ofSeconds(keycloakMaxIdleSeconds)) + .maxLifeTime(Duration.ofSeconds(keycloakMaxLifeSeconds)) + .pendingAcquireTimeout(Duration.ofSeconds(keycloakPendingAcquireTimeoutSeconds)) .evictInBackground(Duration.ofMinutes(1)) .build() val client = HttpClient.create(provider) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10000) - .responseTimeout(Duration.ofSeconds(10)) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, keycloakConnectTimeoutMillis) + .responseTimeout(Duration.ofSeconds(keycloakResponseTimeoutSeconds)) .keepAlive(true) .doOnConnected { it.addHandlerLast(LogbookClientHandler(logbook)) } diff --git a/api/api-app/src/main/resources/application.yml b/api/api-app/src/main/resources/application.yml index ec0a0b34f..aa3606c00 100644 --- a/api/api-app/src/main/resources/application.yml +++ b/api/api-app/src/main/resources/application.yml @@ -107,10 +107,10 @@ logging: level: co.nilin: INFO org.zalando.logbook: TRACE - reactor.netty.pool: DEBUG - reactor.netty.http.client: DEBUG - org.springframework.web.reactive.function.client: DEBUG - co.nilin.opex.api.ports.proxy.impl: DEBUG + reactor.netty.pool: WARN + reactor.netty.http.client: WARN + org.springframework.web.reactive.function.client: WARN + co.nilin.opex.api.ports.proxy.impl: INFO app: base: @@ -152,6 +152,31 @@ app: api: crypto: key: ${api_crypto_key:0e1fd29572ec8c85970d76e3433e96ee} + http: + client: + wiretap: + enabled: ${HTTP_CLIENT_WIRETAP_ENABLED:false} + general: + max-connections: ${HTTP_CLIENT_GENERAL_MAX_CONNECTIONS:300} + pending-acquire-max-count: ${HTTP_CLIENT_GENERAL_PENDING_ACQUIRE_MAX_COUNT:1000} + max-idle-seconds: ${HTTP_CLIENT_GENERAL_MAX_IDLE_SECONDS:30} + max-life-seconds: ${HTTP_CLIENT_GENERAL_MAX_LIFE_SECONDS:120} + pending-acquire-timeout-seconds: ${HTTP_CLIENT_GENERAL_PENDING_ACQUIRE_TIMEOUT_SECONDS:30} + connect-timeout-millis: ${HTTP_CLIENT_GENERAL_CONNECT_TIMEOUT_MILLIS:5000} + response-timeout-seconds: ${HTTP_CLIENT_GENERAL_RESPONSE_TIMEOUT_SECONDS:30} + keycloak: + max-connections: ${HTTP_CLIENT_KEYCLOAK_MAX_CONNECTIONS:150} + pending-acquire-max-count: ${HTTP_CLIENT_KEYCLOAK_PENDING_ACQUIRE_MAX_COUNT:500} + max-idle-seconds: ${HTTP_CLIENT_KEYCLOAK_MAX_IDLE_SECONDS:30} + max-life-seconds: ${HTTP_CLIENT_KEYCLOAK_MAX_LIFE_SECONDS:120} + pending-acquire-timeout-seconds: ${HTTP_CLIENT_KEYCLOAK_PENDING_ACQUIRE_TIMEOUT_SECONDS:60} + connect-timeout-millis: ${HTTP_CLIENT_KEYCLOAK_CONNECT_TIMEOUT_MILLIS:10000} + response-timeout-seconds: ${HTTP_CLIENT_KEYCLOAK_RESPONSE_TIMEOUT_SECONDS:10} + proxy: + market: + max-concurrent-requests: ${API_PROXY_MARKET_MAX_CONCURRENT_REQUESTS:64} + matching: + max-concurrent-requests: ${API_PROXY_MATCHING_MAX_CONCURRENT_REQUESTS:64} cors: enabled: true allowed-origins: ${ALLOWED_ORIGINS:"http://localhost:8110"} @@ -170,4 +195,3 @@ springdoc: display-request-duration: true operations-sorter: method tags-sorter: alpha - diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/config/ProxyDispatchers.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/config/ProxyDispatchers.kt index 1dbbda2aa..48e8aeecd 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/config/ProxyDispatchers.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/config/ProxyDispatchers.kt @@ -4,8 +4,30 @@ import kotlinx.coroutines.reactor.asCoroutineDispatcher import reactor.core.scheduler.Schedulers object ProxyDispatchers { + private fun envInt(name: String, default: Int): Int { + val value = System.getenv(name)?.toIntOrNull() ?: return default + return if (value > 0) value else default + } - val general = Schedulers.newBoundedElastic(8, 16, "general").asCoroutineDispatcher() - val market = Schedulers.newBoundedElastic(8, 16, "market").asCoroutineDispatcher() - val wallet = Schedulers.newBoundedElastic(10, 20, "wallet").asCoroutineDispatcher() + private val cpu = Runtime.getRuntime().availableProcessors().coerceAtLeast(4) + private val defaultThreads = cpu * 4 + private val defaultQueue = 10_000 + + val general = Schedulers.newBoundedElastic( + envInt("API_PROXY_GENERAL_THREADS", defaultThreads), + envInt("API_PROXY_GENERAL_QUEUE", defaultQueue), + "general" + ).asCoroutineDispatcher() + + val market = Schedulers.newBoundedElastic( + envInt("API_PROXY_MARKET_THREADS", defaultThreads), + envInt("API_PROXY_MARKET_QUEUE", defaultQueue), + "market" + ).asCoroutineDispatcher() + + val wallet = Schedulers.newBoundedElastic( + envInt("API_PROXY_WALLET_THREADS", defaultThreads), + envInt("API_PROXY_WALLET_QUEUE", defaultQueue), + "wallet" + ).asCoroutineDispatcher() } \ No newline at end of file diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketUserDataProxyImpl.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketUserDataProxyImpl.kt index b40cfdc03..f67f0c113 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketUserDataProxyImpl.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketUserDataProxyImpl.kt @@ -34,6 +34,9 @@ class MarketUserDataProxyImpl(@Qualifier("generalWebClient") private val webClie @Value("\${app.market.url}") private lateinit var baseUrl: String + + @Value("\${app.proxy.market.max-concurrent-requests:64}") + private var marketMaxConcurrentRequests: Int = 64 private suspend fun retryOnce(backoffMs: Long = 200, block: suspend () -> T): T = try { block() @@ -41,7 +44,9 @@ class MarketUserDataProxyImpl(@Qualifier("generalWebClient") private val webClie delay(backoffMs); block() } - private val mgLimiter = Semaphore(permits = 16, acquiredPermits = 0) + private val mgLimiter by lazy { + Semaphore(permits = marketMaxConcurrentRequests, acquiredPermits = 0) + } override suspend fun queryOrder( token: String, diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MatchingGatewayProxyImpl.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MatchingGatewayProxyImpl.kt index a0718dba0..58ef1533d 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MatchingGatewayProxyImpl.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MatchingGatewayProxyImpl.kt @@ -39,7 +39,13 @@ class MatchingGatewayProxyImpl(@Qualifier("generalWebClient") private val client @Value("\${app.matching-gateway.url}") private lateinit var baseUrl: String - private val mgLimiter = Semaphore(permits = 16, acquiredPermits = 0) // fair-like behavior + + @Value("\${app.proxy.matching.max-concurrent-requests:64}") + private var matchingMaxConcurrentRequests: Int = 64 + + private val mgLimiter by lazy { + Semaphore(permits = matchingMaxConcurrentRequests, acquiredPermits = 0) + } override suspend fun createNewOrder( uuid: String?, pair: String, diff --git a/device-management/pom.xml b/device-management/pom.xml index fc80e43f7..0923d7cdd 100644 --- a/device-management/pom.xml +++ b/device-management/pom.xml @@ -22,7 +22,7 @@ 2.1.0 3.4.2 2024.0.0 - 1.2.25 + 1.2.26 diff --git a/pom.xml b/pom.xml index 853d3c8d5..e49130f86 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 1.9.0 2.7.6 2021.0.5 - 1.2.25 + 1.2.26 1.0.8 true 1.0.1-beta.38 From cfc8456130cb181bf4ec25927cbb08acb876159b Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:18:42 +0200 Subject: [PATCH 15/22] Chore/optimize market accountant --- .../ports/postgres/dao/OrderRepository.kt | 57 +++++++++++++++++++ .../ports/postgres/impl/OrderPersisterImpl.kt | 29 +++++++--- .../ports/postgres/impl/TradePersisterImpl.kt | 5 +- .../ports/postgres/impl/OrderPersisterTest.kt | 49 ++++++++++++++-- 4 files changed, 125 insertions(+), 15 deletions(-) diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt index cc394b310..0b54cbb32 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt @@ -13,6 +13,7 @@ import org.springframework.stereotype.Repository import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import java.math.BigDecimal import java.time.LocalDateTime import java.util.* @@ -49,6 +50,62 @@ interface OrderRepository : ReactiveCrudRepository { updateDate: LocalDateTime = LocalDateTime.now() ): Mono + @Query( + """ + insert into orders ( + ouid, uuid, client_order_id, symbol, order_id, + maker_fee, taker_fee, left_side_fraction, right_side_fraction, + user_level, side, match_constraint, order_type, + price, quantity, quote_quantity, create_date, update_date + ) values ( + :ouid, :uuid, :clientOrderId, :symbol, :orderId, + :makerFee, :takerFee, :leftSideFraction, :rightSideFraction, + :userLevel, :side, :matchConstraint, :orderType, + :price, :quantity, :quoteQuantity, :createDate, :updateDate + ) + on conflict (ouid) do nothing + returning ouid + """ + ) + fun insertOrderIfAbsent( + @Param("ouid") + ouid: String, + @Param("uuid") + uuid: String, + @Param("clientOrderId") + clientOrderId: String?, + @Param("symbol") + symbol: String, + @Param("orderId") + orderId: Long?, + @Param("makerFee") + makerFee: BigDecimal?, + @Param("takerFee") + takerFee: BigDecimal?, + @Param("leftSideFraction") + leftSideFraction: BigDecimal?, + @Param("rightSideFraction") + rightSideFraction: BigDecimal?, + @Param("userLevel") + userLevel: String?, + @Param("side") + side: String?, + @Param("matchConstraint") + matchConstraint: String?, + @Param("orderType") + orderType: String?, + @Param("price") + price: BigDecimal?, + @Param("quantity") + quantity: BigDecimal?, + @Param("quoteQuantity") + quoteQuantity: BigDecimal?, + @Param("createDate") + createDate: LocalDateTime?, + @Param("updateDate") + updateDate: LocalDateTime + ): Mono + @Query( """ select * from orders diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt index 87a7d7a3d..a4fd7dda7 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterImpl.kt @@ -17,8 +17,6 @@ import kotlinx.coroutines.reactive.awaitFirstOrNull import kotlinx.coroutines.reactor.awaitSingle import kotlinx.coroutines.reactor.awaitSingleOrNull import org.slf4j.LoggerFactory -import org.springframework.dao.DataIntegrityViolationException -import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -57,12 +55,27 @@ class OrderPersisterImpl( LocalDateTime.now(), LocalDateTime.now() ) - try { - orderRepository.save(orderModel).awaitFirstOrNull() - } catch (e: DuplicateKeyException) { - logger.info("order ${order.ouid} is duplicate; skipping create flow") - return - } catch (e: DataIntegrityViolationException) { + val inserted = orderRepository.insertOrderIfAbsent( + ouid = orderModel.ouid, + uuid = orderModel.uuid, + clientOrderId = orderModel.clientOrderId, + symbol = orderModel.symbol, + orderId = orderModel.orderId, + makerFee = orderModel.makerFee, + takerFee = orderModel.takerFee, + leftSideFraction = orderModel.leftSideFraction, + rightSideFraction = orderModel.rightSideFraction, + userLevel = orderModel.userLevel, + side = orderModel.direction?.name, + matchConstraint = orderModel.constraint?.name, + orderType = orderModel.type?.name, + price = orderModel.price, + quantity = orderModel.quantity, + quoteQuantity = orderModel.quoteQuantity, + createDate = orderModel.createDate, + updateDate = orderModel.updateDate + ).awaitFirstOrNull() != null + if (!inserted) { logger.info("order ${order.ouid} is duplicate; skipping create flow") return } diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt index a257a1d99..d3daa09f7 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/TradePersisterImpl.kt @@ -14,8 +14,10 @@ import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Component import java.time.LocalDateTime import java.time.ZoneId +import java.time.temporal.ChronoUnit import java.util.* import java.util.concurrent.atomic.AtomicLong +import kotlin.math.abs @Component class TradePersisterImpl( @@ -144,11 +146,12 @@ class TradePersisterImpl( } private fun isSameTradePayload(existing: TradeModel, incoming: TradeModel): Boolean { + val tradeDateDeltaSeconds = abs(ChronoUnit.SECONDS.between(existing.tradeDate, incoming.tradeDate)) return existing.makerOuid == incoming.makerOuid && existing.takerOuid == incoming.takerOuid && existing.matchedPrice.compareTo(incoming.matchedPrice) == 0 && existing.matchedQuantity.compareTo(incoming.matchedQuantity) == 0 && - existing.tradeDate == incoming.tradeDate && + tradeDateDeltaSeconds <= 5 && existing.makerCommission == incoming.makerCommission && existing.takerCommission == incoming.takerCommission && existing.makerCommissionAsset == incoming.makerCommissionAsset && diff --git a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt index ba1077369..9ab3a2a9f 100644 --- a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt +++ b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/OrderPersisterTest.kt @@ -35,11 +35,31 @@ class OrderPersisterTest { @Test fun givenOrderRepo_whenSaveRichOrder_thenSuccess(): Unit = runBlocking { every { - orderRepository.save(any()) - } returns Mono.just(VALID.MAKER_ORDER_MODEL) + orderRepository.insertOrderIfAbsent( + ouid = any(), + uuid = any(), + clientOrderId = any(), + symbol = any(), + orderId = any(), + makerFee = any(), + takerFee = any(), + leftSideFraction = any(), + rightSideFraction = any(), + userLevel = any(), + side = any(), + matchConstraint = any(), + orderType = any(), + price = any(), + quantity = any(), + quoteQuantity = any(), + createDate = any(), + updateDate = any() + ) + } returns Mono.just(VALID.RICH_ORDER.ouid) every { orderStatusRepository.insert(any(), any(), any(), any(), any(), any()) } returns Mono.empty() + every { orderStatusRepository.findMostRecentByOUID(any()) } returns Mono.just(VALID.MAKER_ORDER_STATUS_MODEL) @@ -86,10 +106,27 @@ class OrderPersisterTest { @Test fun givenDuplicateOrderCreate_whenSaveRichOrder_thenIgnoredAsIdempotent(): Unit = runBlocking { every { - orderRepository.save(any()) - } returns Mono.error(DuplicateKeyException("duplicate order")) - - assertThatNoException().isThrownBy { runBlocking { orderPersister.save(VALID.RICH_ORDER) } } + orderRepository.insertOrderIfAbsent( + ouid = any(), + uuid = any(), + clientOrderId = any(), + symbol = any(), + orderId = any(), + makerFee = any(), + takerFee = any(), + leftSideFraction = any(), + rightSideFraction = any(), + userLevel = any(), + side = any(), + matchConstraint = any(), + orderType = any(), + price = any(), + quantity = any(), + quoteQuantity = any(), + createDate = any(), + updateDate = any() + ) + } returns Mono.empty() verify(exactly = 0) { orderStatusRepository.insert(any(), any(), any(), any(), any(), any()) From 8a9c4bdea9867f57e93e6f18c95e52fffdecf26d Mon Sep 17 00:00:00 2001 From: fatemeh imanipour Date: Thu, 13 Aug 2026 17:07:53 +0330 Subject: [PATCH 16/22] Fix a merge conflict --- .../scheduler/FinancialActionsArchiveJob.kt | 40 ++++++++++ .../src/main/resources/application.yml | 10 +++ .../scheduler/FinancialActionJobManagerIT.kt | 3 +- .../service/FinancialActionJobManagerImpl.kt | 9 +-- .../core/spi/FinancialActionPersister.kt | 3 + .../postgres/dao/FinancialActionRepository.kt | 78 ++++++++++++++++++- .../impl/FinancialActionLoaderImpl.kt | 41 +++++----- .../impl/FinancialActionPersisterImpl.kt | 4 + .../src/main/resources/schema.sql | 59 ++++++++++++++ .../walletproxy/proxy/WalletProxyImpl.kt | 53 ++++++++++--- .../walletproxy/proxy/WalletProxyImplTest.kt | 5 +- .../ports/postgres/dao/OrderRepository.kt | 56 +++++++++++++ 12 files changed, 320 insertions(+), 41 deletions(-) create mode 100644 accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt diff --git a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt new file mode 100644 index 000000000..f0d26804b --- /dev/null +++ b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt @@ -0,0 +1,40 @@ +package co.nilin.opex.accountant.app.scheduler + +import co.nilin.opex.accountant.core.spi.FinancialActionPersister +import kotlinx.coroutines.runBlocking +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Profile +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import java.time.LocalDateTime + +@Service +@Profile("scheduled") +class FinancialActionsArchiveJob( + private val financialActionPersister: FinancialActionPersister +) { + private val log = LoggerFactory.getLogger(FinancialActionsArchiveJob::class.java) + + @Value("\${app.fi-action.archive.enabled:true}") + private var enabled: Boolean = true + + @Value("\${app.fi-action.archive.retention-days:30}") + private var retentionDays: Long = 30 + + @Value("\${app.fi-action.archive.batch-size:1000}") + private var batchSize: Int = 1000 + + @Scheduled(fixedDelayString = "\${app.fi-action.archive.fixed-delay-ms:300000}", initialDelay = 60000) + fun archiveProcessedActions() { + if (!enabled || batchSize <= 0 || retentionDays <= 0) return + + runBlocking { + val before = LocalDateTime.now().minusDays(retentionDays) + val archived = financialActionPersister.archiveProcessedActions(before, batchSize) + if (archived > 0) { + log.info("Archived $archived processed financial actions older than $before") + } + } + } +} diff --git a/accountant/accountant-app/src/main/resources/application.yml b/accountant/accountant-app/src/main/resources/application.yml index f191ca34c..3c8b8c1d4 100644 --- a/accountant/accountant-app/src/main/resources/application.yml +++ b/accountant/accountant-app/src/main/resources/application.yml @@ -101,11 +101,21 @@ app: address: 1 wallet: url: lb://opex-wallet/ + http: + retry: + count: 2 + delay-millis: 250 + timeout-seconds: 10 fi-action: retry: count: 10 delay-seconds: 5 delay-multiplier: 3 + archive: + enabled: true + retention-days: 30 + batch-size: 1000 + fixed-delay-ms: 300000 zone-offset: +03:30 trade-volume-calculation-currency: ${TRADE_VOLUME_CALCULATION_CURRENCY:USDT} withdraw-volume-calculation-currency: ${WITHDRAW_VOLUME_CALCULATION_CURRENCY:USDT} diff --git a/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt b/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt index c29f3c13b..2081965d6 100644 --- a/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt +++ b/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt @@ -14,7 +14,6 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.mockito.InOrder import org.mockito.Mockito -import org.mockito.Mockito.any import org.mockito.Mockito.`when` import org.mockito.kotlin.eq import org.springframework.beans.factory.annotation.Autowired @@ -248,7 +247,7 @@ class FinancialActionJobManagerIT : KafkaEnabledTest() { eq(fi.receiver), eq(fi.amount), eq(fi.eventType + fi.pointer), - any(), + eq("accountant:fiActions:${fi.uuid}"), eq(fi.category.toString()), ) } diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt index 0dce40930..125ee32d2 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt @@ -23,13 +23,6 @@ class FinancialActionJobManagerImpl( .also { if (it.isNotEmpty()) logger.info("Processing ${it.size} financial actions") } .forEach { try { - if (it.parent != null) { - val reloadParent = financialActionLoader.loadFinancialAction(it.parent.id)!! - if (reloadParent.status != FinancialActionStatus.PROCESSED) { - logger.warn("Financial job (uuid=${it.uuid}) skipped because of parent status: uuid=${reloadParent.uuid}, status=${reloadParent.status}") - return@forEach - } - } walletProxy.transfer( it.symbol, it.senderWalletType, @@ -70,7 +63,7 @@ class FinancialActionJobManagerImpl( it.receiver, it.amount, it.eventType + it.pointer, - "accountant:fiActions:${it.id.toString()}", + "accountant:fiActions:${it.uuid}", it.category.toString() ) with(financialActionPersister) { diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt index d8747ced3..a4e4afa2e 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt @@ -2,6 +2,7 @@ package co.nilin.opex.accountant.core.spi import co.nilin.opex.accountant.core.model.FinancialAction import co.nilin.opex.accountant.core.model.FinancialActionStatus +import java.time.LocalDateTime interface FinancialActionPersister { @@ -20,4 +21,6 @@ interface FinancialActionPersister { suspend fun updateStatusNewTx(financialAction: FinancialAction, status: FinancialActionStatus) suspend fun retrySuccessful(financialAction: FinancialAction) + + suspend fun archiveProcessedActions(before: LocalDateTime, limit: Int): Int } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt index 3139f647d..16dbeddf4 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt @@ -10,6 +10,7 @@ import org.springframework.data.repository.reactive.ReactiveCrudRepository import org.springframework.stereotype.Repository import reactor.core.publisher.Mono import java.math.BigDecimal +import java.time.LocalDateTime @Repository interface FinancialActionRepository : ReactiveCrudRepository { @@ -45,8 +46,83 @@ interface FinancialActionRepository : ReactiveCrudRepository + + @Query( + """ + with candidates as ( + select id + from fi_actions + where status = 'PROCESSED' + and create_date < :before + and not exists ( + select 1 from fi_action_retry far + where far.fa_id = fi_actions.id and far.is_resolved = false + ) + order by create_date + limit :limit + ), + moved_actions as ( + insert into fi_actions_archive ( + id, uuid, parent_id, event_type, pointer, symbol, amount, sender, sender_wallet_type, + receiver, receiver_wallet_type, agent, ip, create_date, status, category_name + ) + select fa.id, fa.uuid, fa.parent_id, fa.event_type, fa.pointer, fa.symbol, fa.amount, fa.sender, fa.sender_wallet_type, + fa.receiver, fa.receiver_wallet_type, fa.agent, fa.ip, fa.create_date, fa.status, fa.category_name + from fi_actions fa + join candidates c on c.id = fa.id + on conflict (id) do nothing + returning id + ), + moved_retries as ( + insert into fi_action_retry_archive (id, fa_id, retries, next_run_time, is_resolved, has_given_up) + select far.id, far.fa_id, far.retries, far.next_run_time, far.is_resolved, far.has_given_up + from fi_action_retry far + join moved_actions ma on ma.id = far.fa_id + on conflict (id) do nothing + returning id + ), + moved_errors as ( + insert into fi_action_error_archive (id, fa_id, error, message, body, retry_id, date) + select fae.id, fae.fa_id, fae.error, fae.message, fae.body, fae.retry_id, fae.date + from fi_action_error fae + join moved_actions ma on ma.id = fae.fa_id + on conflict (id) do nothing + returning id + ), + deleted_errors as ( + delete from fi_action_error fae + using moved_actions ma + where fae.fa_id = ma.id + returning fae.id + ), + deleted_retries as ( + delete from fi_action_retry far + using moved_actions ma + where far.fa_id = ma.id + returning far.id + ), + deleted_actions as ( + delete from fi_actions fa + using moved_actions ma + where fa.id = ma.id + returning fa.id + ) + select count(1) from deleted_actions + """ + ) + fun archiveProcessedActions( + @Param("before") before: LocalDateTime, + @Param("limit") limit: Int + ): Mono } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt index 32bf2cfd5..1fb65e088 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt @@ -6,6 +6,7 @@ import co.nilin.opex.accountant.core.spi.FinancialActionLoader import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionErrorRepository import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionRepository import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionRetryRepository +import co.nilin.opex.accountant.ports.postgres.model.FinancialActionModel import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList @@ -35,7 +36,7 @@ class FinancialActionLoaderImpl( override suspend fun loadReadyToProcess(offset: Long, size: Long): List { return financialActionRepository.findReadyToProcess( PageRequest.of(offset.toInt(), size.toInt(), Sort.by(Sort.Direction.ASC, "createDate")) - ).map { loadFinancialAction(it.id)!! } + ).map { mapToFinancialAction(it) } .toList() } @@ -80,25 +81,27 @@ class FinancialActionLoaderImpl( override suspend fun loadRetries(limit: Int): List { return faRetryRepository.findAllRetries(LocalDateTime.now(), limit) - .map { - FinancialAction( - null, // Skipping parent. If it's in retry, it means its parent is already processed - it.eventType, - it.pointer, - it.symbol, - it.amount, - it.sender, - it.senderWalletType, - it.receiver, - it.receiverWalletType, - it.createDate, - it.categoryName, - it.status, - it.uuid, - it.id - ) - } + .map { mapToFinancialAction(it) } .collectList() .awaitFirstOrElse { emptyList() } } + + private fun mapToFinancialAction(financialAction: FinancialActionModel): FinancialAction { + return FinancialAction( + null, + financialAction.eventType, + financialAction.pointer, + financialAction.symbol, + financialAction.amount, + financialAction.sender, + financialAction.senderWalletType, + financialAction.receiver, + financialAction.receiverWalletType, + financialAction.createDate, + financialAction.categoryName, + financialAction.status, + financialAction.uuid, + financialAction.id + ) + } } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt index 86d9dcd89..12292c2e6 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt @@ -131,6 +131,10 @@ class FinancialActionPersisterImpl( faRetryRepository.updateResolvedTrue(financialAction.id!!).awaitSingleOrNull() } + override suspend fun archiveProcessedActions(before: LocalDateTime, limit: Int): Int { + return (repository.archiveProcessedActions(before, limit).awaitSingleOrNull() ?: 0).toInt() + } + override suspend fun updateStatus(faUuid: String, status: FinancialActionStatus) { repository.updateStatus(faUuid, status).awaitSingleOrNull() } diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql index c4d6999c6..8b7b1b772 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql @@ -49,6 +49,8 @@ CREATE INDEX IF NOT EXISTS idx_fi_actions_symbol ON fi_actions (symbol); CREATE INDEX IF NOT EXISTS idx_fi_event_type ON fi_actions (event_type); CREATE INDEX IF NOT EXISTS idx_fi_actions_status ON fi_actions (status); CREATE INDEX IF NOT EXISTS idx_fi_actions_pointer ON fi_actions (pointer); +CREATE INDEX IF NOT EXISTS idx_fi_actions_status_create_date ON fi_actions (status, create_date); +CREATE INDEX IF NOT EXISTS idx_fi_actions_parent_status ON fi_actions (parent_id, status); ALTER TABLE fi_actions ADD COLUMN IF NOT EXISTS category_name VARCHAR(36); @@ -63,6 +65,11 @@ CREATE TABLE IF NOT EXISTS fi_action_retry has_given_up BOOLEAN NOT NULL DEFAULT false ); +CREATE INDEX IF NOT EXISTS idx_fi_action_retry_due + ON fi_action_retry (next_run_time) + WHERE has_given_up = false + AND is_resolved = false; + CREATE TABLE IF NOT EXISTS fi_action_error ( id SERIAL PRIMARY KEY, @@ -74,6 +81,58 @@ CREATE TABLE IF NOT EXISTS fi_action_error date TIMESTAMP NOT NULL DEFAULT CURRENT_DATE ); +CREATE INDEX IF NOT EXISTS idx_fi_action_error_fa_id_date ON fi_action_error (fa_id, date); + +CREATE TABLE IF NOT EXISTS fi_actions_archive +( + id INTEGER PRIMARY KEY, + uuid VARCHAR(72) NOT NULL UNIQUE, + parent_id INTEGER, + event_type VARCHAR(72) NOT NULL, + pointer VARCHAR(72) NOT NULL, + symbol VARCHAR(36) NOT NULL, + amount DECIMAL NOT NULL, + sender VARCHAR(36) NOT NULL, + sender_wallet_type VARCHAR(36) NOT NULL, + receiver VARCHAR(36) NOT NULL, + receiver_wallet_type VARCHAR(36) NOT NULL, + agent VARCHAR(20), + ip VARCHAR(11), + create_date TIMESTAMP NOT NULL, + status VARCHAR(20), + category_name VARCHAR(36), + archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_fi_actions_archive_create_date ON fi_actions_archive (create_date); + +CREATE TABLE IF NOT EXISTS fi_action_retry_archive +( + id INTEGER PRIMARY KEY, + fa_id INTEGER NOT NULL UNIQUE, + retries INTEGER NOT NULL DEFAULT 0, + next_run_time TIMESTAMP NOT NULL, + is_resolved BOOLEAN NOT NULL DEFAULT false, + has_given_up BOOLEAN NOT NULL DEFAULT false, + archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_fi_action_retry_archive_fa_id ON fi_action_retry_archive (fa_id); + +CREATE TABLE IF NOT EXISTS fi_action_error_archive +( + id INTEGER PRIMARY KEY, + fa_id INTEGER NOT NULL, + error TEXT NOT NULL, + message TEXT NOT NULL, + body TEXT, + retry_id INTEGER, + date TIMESTAMP NOT NULL, + archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_fi_action_error_archive_fa_id ON fi_action_error_archive (fa_id); + CREATE TABLE IF NOT EXISTS pair_config ( pair VARCHAR(72) PRIMARY KEY, diff --git a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt index ff7078afd..c3701d421 100644 --- a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt +++ b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt @@ -11,15 +11,24 @@ import kotlinx.coroutines.reactive.awaitFirstOrNull import org.springframework.beans.factory.annotation.Value import org.springframework.http.MediaType import org.springframework.stereotype.Component +import org.springframework.web.reactive.function.client.WebClientRequestException import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.bodyToMono +import reactor.core.publisher.Mono +import reactor.util.retry.Retry +import java.io.IOException +import java.net.ConnectException import java.math.BigDecimal +import java.time.Duration +import java.util.concurrent.TimeoutException @Component class WalletProxyImpl( private val webClient: WebClient, - @Value("\${app.wallet.url}") - private val walletBaseUrl: String + @Value("\${app.wallet.url}") private val walletBaseUrl: String, + @Value("\${app.wallet.http.retry.count:2}") private val retryCount: Long = 2, + @Value("\${app.wallet.http.retry.delay-millis:250}") private val retryDelayMillis: Long = 250, + @Value("\${app.wallet.http.timeout-seconds:10}") private val timeoutSeconds: Long = 10 ) : WalletProxy { data class TransferBody( @@ -39,14 +48,15 @@ class WalletProxyImpl( transferRef: String?, transferCategory: String ) { - webClient.post() - .uri("$walletBaseUrl/v2/transfer/${amount}_$symbol/from/${senderUuid}_$senderWalletType/to/${receiverUuid}_$receiverWalletType") - .contentType(MediaType.APPLICATION_JSON) - .bodyValue(TransferBody(description, transferRef, transferCategory)) - .retrieve() - .onStatus({ t -> t.isError }, { it.createException() }) - .bodyToMono() - .awaitFirst() + withTransientRetry { + webClient.post() + .uri("$walletBaseUrl/v2/transfer/${amount}_$symbol/from/${senderUuid}_$senderWalletType/to/${receiverUuid}_$receiverWalletType") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(TransferBody(description, transferRef, transferCategory)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + }.awaitFirst() } override suspend fun canFulfil(symbol: String, walletType: WalletType, uuid: String, amount: BigDecimal): Boolean { @@ -56,6 +66,7 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono() + .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirst() .result } @@ -69,6 +80,7 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono() + .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirstOrNull() } @@ -79,6 +91,27 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono>() + .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirst() } + + private fun withTransientRetry(request: () -> Mono): Mono { + return request() + .timeout(Duration.ofSeconds(timeoutSeconds)) + .retryWhen( + Retry.backoff(retryCount, Duration.ofMillis(retryDelayMillis)) + .filter { error -> + when { + error is WebClientRequestException -> true + error is TimeoutException -> true + error is ConnectException -> true + error is IOException -> true + error.cause is TimeoutException -> true + error.cause is ConnectException -> true + else -> false + } + } + .onRetryExhaustedThrow { _, signal -> signal.failure() } + ) + } } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt b/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt index 107c2b794..0fe54ce0d 100644 --- a/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt +++ b/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt @@ -22,7 +22,10 @@ class WalletProxyImplTest { private lateinit var mockServer: MockServerClient private val walletProxyImpl = WalletProxyImpl( WebClient.builder().build(), - "http://localhost:8089" + "http://localhost:8089", + 1, + 10, + 5 ) private val objectMapper = ObjectMapper() diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt index 297054f8a..0b54cbb32 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/OrderRepository.kt @@ -50,6 +50,62 @@ interface OrderRepository : ReactiveCrudRepository { updateDate: LocalDateTime = LocalDateTime.now() ): Mono + @Query( + """ + insert into orders ( + ouid, uuid, client_order_id, symbol, order_id, + maker_fee, taker_fee, left_side_fraction, right_side_fraction, + user_level, side, match_constraint, order_type, + price, quantity, quote_quantity, create_date, update_date + ) values ( + :ouid, :uuid, :clientOrderId, :symbol, :orderId, + :makerFee, :takerFee, :leftSideFraction, :rightSideFraction, + :userLevel, :side, :matchConstraint, :orderType, + :price, :quantity, :quoteQuantity, :createDate, :updateDate + ) + on conflict (ouid) do nothing + returning ouid + """ + ) + fun insertOrderIfAbsent( + @Param("ouid") + ouid: String, + @Param("uuid") + uuid: String, + @Param("clientOrderId") + clientOrderId: String?, + @Param("symbol") + symbol: String, + @Param("orderId") + orderId: Long?, + @Param("makerFee") + makerFee: BigDecimal?, + @Param("takerFee") + takerFee: BigDecimal?, + @Param("leftSideFraction") + leftSideFraction: BigDecimal?, + @Param("rightSideFraction") + rightSideFraction: BigDecimal?, + @Param("userLevel") + userLevel: String?, + @Param("side") + side: String?, + @Param("matchConstraint") + matchConstraint: String?, + @Param("orderType") + orderType: String?, + @Param("price") + price: BigDecimal?, + @Param("quantity") + quantity: BigDecimal?, + @Param("quoteQuantity") + quoteQuantity: BigDecimal?, + @Param("createDate") + createDate: LocalDateTime?, + @Param("updateDate") + updateDate: LocalDateTime + ): Mono + @Query( """ select * from orders From 2cc035d9f72747dc53063cd779bc7456c6727920 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:31:42 +0330 Subject: [PATCH 17/22] Update error-handler version to 1.2.27 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e49130f86..b05355151 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 1.9.0 2.7.6 2021.0.5 - 1.2.26 + 1.2.27 1.0.8 true 1.0.1-beta.38 From 430f9acd5feae355dbf8752e87a951699105e5e3 Mon Sep 17 00:00:00 2001 From: fatemeh imanipour Date: Thu, 13 Aug 2026 18:42:14 +0330 Subject: [PATCH 18/22] Stash all changes about fi actions --- .../scheduler/FinancialActionsArchiveJob.kt | 40 ---------- .../src/main/resources/application.yml | 10 --- .../scheduler/FinancialActionJobManagerIT.kt | 3 +- .../service/FinancialActionJobManagerImpl.kt | 9 ++- .../core/spi/FinancialActionPersister.kt | 3 - .../postgres/dao/FinancialActionRepository.kt | 78 +------------------ .../impl/FinancialActionLoaderImpl.kt | 41 +++++----- .../impl/FinancialActionPersisterImpl.kt | 4 - .../src/main/resources/schema.sql | 59 -------------- .../walletproxy/proxy/WalletProxyImpl.kt | 53 +++---------- .../walletproxy/proxy/WalletProxyImplTest.kt | 5 +- device-management/pom.xml | 2 +- 12 files changed, 42 insertions(+), 265 deletions(-) delete mode 100644 accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt diff --git a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt deleted file mode 100644 index f0d26804b..000000000 --- a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt +++ /dev/null @@ -1,40 +0,0 @@ -package co.nilin.opex.accountant.app.scheduler - -import co.nilin.opex.accountant.core.spi.FinancialActionPersister -import kotlinx.coroutines.runBlocking -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Profile -import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Service -import java.time.LocalDateTime - -@Service -@Profile("scheduled") -class FinancialActionsArchiveJob( - private val financialActionPersister: FinancialActionPersister -) { - private val log = LoggerFactory.getLogger(FinancialActionsArchiveJob::class.java) - - @Value("\${app.fi-action.archive.enabled:true}") - private var enabled: Boolean = true - - @Value("\${app.fi-action.archive.retention-days:30}") - private var retentionDays: Long = 30 - - @Value("\${app.fi-action.archive.batch-size:1000}") - private var batchSize: Int = 1000 - - @Scheduled(fixedDelayString = "\${app.fi-action.archive.fixed-delay-ms:300000}", initialDelay = 60000) - fun archiveProcessedActions() { - if (!enabled || batchSize <= 0 || retentionDays <= 0) return - - runBlocking { - val before = LocalDateTime.now().minusDays(retentionDays) - val archived = financialActionPersister.archiveProcessedActions(before, batchSize) - if (archived > 0) { - log.info("Archived $archived processed financial actions older than $before") - } - } - } -} diff --git a/accountant/accountant-app/src/main/resources/application.yml b/accountant/accountant-app/src/main/resources/application.yml index 3c8b8c1d4..f191ca34c 100644 --- a/accountant/accountant-app/src/main/resources/application.yml +++ b/accountant/accountant-app/src/main/resources/application.yml @@ -101,21 +101,11 @@ app: address: 1 wallet: url: lb://opex-wallet/ - http: - retry: - count: 2 - delay-millis: 250 - timeout-seconds: 10 fi-action: retry: count: 10 delay-seconds: 5 delay-multiplier: 3 - archive: - enabled: true - retention-days: 30 - batch-size: 1000 - fixed-delay-ms: 300000 zone-offset: +03:30 trade-volume-calculation-currency: ${TRADE_VOLUME_CALCULATION_CURRENCY:USDT} withdraw-volume-calculation-currency: ${WITHDRAW_VOLUME_CALCULATION_CURRENCY:USDT} diff --git a/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt b/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt index 2081965d6..c29f3c13b 100644 --- a/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt +++ b/accountant/accountant-app/src/test/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionJobManagerIT.kt @@ -14,6 +14,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.mockito.InOrder import org.mockito.Mockito +import org.mockito.Mockito.any import org.mockito.Mockito.`when` import org.mockito.kotlin.eq import org.springframework.beans.factory.annotation.Autowired @@ -247,7 +248,7 @@ class FinancialActionJobManagerIT : KafkaEnabledTest() { eq(fi.receiver), eq(fi.amount), eq(fi.eventType + fi.pointer), - eq("accountant:fiActions:${fi.uuid}"), + any(), eq(fi.category.toString()), ) } diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt index 125ee32d2..0dce40930 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/service/FinancialActionJobManagerImpl.kt @@ -23,6 +23,13 @@ class FinancialActionJobManagerImpl( .also { if (it.isNotEmpty()) logger.info("Processing ${it.size} financial actions") } .forEach { try { + if (it.parent != null) { + val reloadParent = financialActionLoader.loadFinancialAction(it.parent.id)!! + if (reloadParent.status != FinancialActionStatus.PROCESSED) { + logger.warn("Financial job (uuid=${it.uuid}) skipped because of parent status: uuid=${reloadParent.uuid}, status=${reloadParent.status}") + return@forEach + } + } walletProxy.transfer( it.symbol, it.senderWalletType, @@ -63,7 +70,7 @@ class FinancialActionJobManagerImpl( it.receiver, it.amount, it.eventType + it.pointer, - "accountant:fiActions:${it.uuid}", + "accountant:fiActions:${it.id.toString()}", it.category.toString() ) with(financialActionPersister) { diff --git a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt index a4e4afa2e..d8747ced3 100644 --- a/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt +++ b/accountant/accountant-core/src/main/kotlin/co/nilin/opex/accountant/core/spi/FinancialActionPersister.kt @@ -2,7 +2,6 @@ package co.nilin.opex.accountant.core.spi import co.nilin.opex.accountant.core.model.FinancialAction import co.nilin.opex.accountant.core.model.FinancialActionStatus -import java.time.LocalDateTime interface FinancialActionPersister { @@ -21,6 +20,4 @@ interface FinancialActionPersister { suspend fun updateStatusNewTx(financialAction: FinancialAction, status: FinancialActionStatus) suspend fun retrySuccessful(financialAction: FinancialAction) - - suspend fun archiveProcessedActions(before: LocalDateTime, limit: Int): Int } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt index 16dbeddf4..3139f647d 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt @@ -10,7 +10,6 @@ import org.springframework.data.repository.reactive.ReactiveCrudRepository import org.springframework.stereotype.Repository import reactor.core.publisher.Mono import java.math.BigDecimal -import java.time.LocalDateTime @Repository interface FinancialActionRepository : ReactiveCrudRepository { @@ -46,83 +45,8 @@ interface FinancialActionRepository : ReactiveCrudRepository - - @Query( - """ - with candidates as ( - select id - from fi_actions - where status = 'PROCESSED' - and create_date < :before - and not exists ( - select 1 from fi_action_retry far - where far.fa_id = fi_actions.id and far.is_resolved = false - ) - order by create_date - limit :limit - ), - moved_actions as ( - insert into fi_actions_archive ( - id, uuid, parent_id, event_type, pointer, symbol, amount, sender, sender_wallet_type, - receiver, receiver_wallet_type, agent, ip, create_date, status, category_name - ) - select fa.id, fa.uuid, fa.parent_id, fa.event_type, fa.pointer, fa.symbol, fa.amount, fa.sender, fa.sender_wallet_type, - fa.receiver, fa.receiver_wallet_type, fa.agent, fa.ip, fa.create_date, fa.status, fa.category_name - from fi_actions fa - join candidates c on c.id = fa.id - on conflict (id) do nothing - returning id - ), - moved_retries as ( - insert into fi_action_retry_archive (id, fa_id, retries, next_run_time, is_resolved, has_given_up) - select far.id, far.fa_id, far.retries, far.next_run_time, far.is_resolved, far.has_given_up - from fi_action_retry far - join moved_actions ma on ma.id = far.fa_id - on conflict (id) do nothing - returning id - ), - moved_errors as ( - insert into fi_action_error_archive (id, fa_id, error, message, body, retry_id, date) - select fae.id, fae.fa_id, fae.error, fae.message, fae.body, fae.retry_id, fae.date - from fi_action_error fae - join moved_actions ma on ma.id = fae.fa_id - on conflict (id) do nothing - returning id - ), - deleted_errors as ( - delete from fi_action_error fae - using moved_actions ma - where fae.fa_id = ma.id - returning fae.id - ), - deleted_retries as ( - delete from fi_action_retry far - using moved_actions ma - where far.fa_id = ma.id - returning far.id - ), - deleted_actions as ( - delete from fi_actions fa - using moved_actions ma - where fa.id = ma.id - returning fa.id - ) - select count(1) from deleted_actions - """ - ) - fun archiveProcessedActions( - @Param("before") before: LocalDateTime, - @Param("limit") limit: Int - ): Mono } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt index 1fb65e088..32bf2cfd5 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt @@ -6,7 +6,6 @@ import co.nilin.opex.accountant.core.spi.FinancialActionLoader import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionErrorRepository import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionRepository import co.nilin.opex.accountant.ports.postgres.dao.FinancialActionRetryRepository -import co.nilin.opex.accountant.ports.postgres.model.FinancialActionModel import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList @@ -36,7 +35,7 @@ class FinancialActionLoaderImpl( override suspend fun loadReadyToProcess(offset: Long, size: Long): List { return financialActionRepository.findReadyToProcess( PageRequest.of(offset.toInt(), size.toInt(), Sort.by(Sort.Direction.ASC, "createDate")) - ).map { mapToFinancialAction(it) } + ).map { loadFinancialAction(it.id)!! } .toList() } @@ -81,27 +80,25 @@ class FinancialActionLoaderImpl( override suspend fun loadRetries(limit: Int): List { return faRetryRepository.findAllRetries(LocalDateTime.now(), limit) - .map { mapToFinancialAction(it) } + .map { + FinancialAction( + null, // Skipping parent. If it's in retry, it means its parent is already processed + it.eventType, + it.pointer, + it.symbol, + it.amount, + it.sender, + it.senderWalletType, + it.receiver, + it.receiverWalletType, + it.createDate, + it.categoryName, + it.status, + it.uuid, + it.id + ) + } .collectList() .awaitFirstOrElse { emptyList() } } - - private fun mapToFinancialAction(financialAction: FinancialActionModel): FinancialAction { - return FinancialAction( - null, - financialAction.eventType, - financialAction.pointer, - financialAction.symbol, - financialAction.amount, - financialAction.sender, - financialAction.senderWalletType, - financialAction.receiver, - financialAction.receiverWalletType, - financialAction.createDate, - financialAction.categoryName, - financialAction.status, - financialAction.uuid, - financialAction.id - ) - } } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt index 12292c2e6..86d9dcd89 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt @@ -131,10 +131,6 @@ class FinancialActionPersisterImpl( faRetryRepository.updateResolvedTrue(financialAction.id!!).awaitSingleOrNull() } - override suspend fun archiveProcessedActions(before: LocalDateTime, limit: Int): Int { - return (repository.archiveProcessedActions(before, limit).awaitSingleOrNull() ?: 0).toInt() - } - override suspend fun updateStatus(faUuid: String, status: FinancialActionStatus) { repository.updateStatus(faUuid, status).awaitSingleOrNull() } diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql index 8b7b1b772..c4d6999c6 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql @@ -49,8 +49,6 @@ CREATE INDEX IF NOT EXISTS idx_fi_actions_symbol ON fi_actions (symbol); CREATE INDEX IF NOT EXISTS idx_fi_event_type ON fi_actions (event_type); CREATE INDEX IF NOT EXISTS idx_fi_actions_status ON fi_actions (status); CREATE INDEX IF NOT EXISTS idx_fi_actions_pointer ON fi_actions (pointer); -CREATE INDEX IF NOT EXISTS idx_fi_actions_status_create_date ON fi_actions (status, create_date); -CREATE INDEX IF NOT EXISTS idx_fi_actions_parent_status ON fi_actions (parent_id, status); ALTER TABLE fi_actions ADD COLUMN IF NOT EXISTS category_name VARCHAR(36); @@ -65,11 +63,6 @@ CREATE TABLE IF NOT EXISTS fi_action_retry has_given_up BOOLEAN NOT NULL DEFAULT false ); -CREATE INDEX IF NOT EXISTS idx_fi_action_retry_due - ON fi_action_retry (next_run_time) - WHERE has_given_up = false - AND is_resolved = false; - CREATE TABLE IF NOT EXISTS fi_action_error ( id SERIAL PRIMARY KEY, @@ -81,58 +74,6 @@ CREATE TABLE IF NOT EXISTS fi_action_error date TIMESTAMP NOT NULL DEFAULT CURRENT_DATE ); -CREATE INDEX IF NOT EXISTS idx_fi_action_error_fa_id_date ON fi_action_error (fa_id, date); - -CREATE TABLE IF NOT EXISTS fi_actions_archive -( - id INTEGER PRIMARY KEY, - uuid VARCHAR(72) NOT NULL UNIQUE, - parent_id INTEGER, - event_type VARCHAR(72) NOT NULL, - pointer VARCHAR(72) NOT NULL, - symbol VARCHAR(36) NOT NULL, - amount DECIMAL NOT NULL, - sender VARCHAR(36) NOT NULL, - sender_wallet_type VARCHAR(36) NOT NULL, - receiver VARCHAR(36) NOT NULL, - receiver_wallet_type VARCHAR(36) NOT NULL, - agent VARCHAR(20), - ip VARCHAR(11), - create_date TIMESTAMP NOT NULL, - status VARCHAR(20), - category_name VARCHAR(36), - archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX IF NOT EXISTS idx_fi_actions_archive_create_date ON fi_actions_archive (create_date); - -CREATE TABLE IF NOT EXISTS fi_action_retry_archive -( - id INTEGER PRIMARY KEY, - fa_id INTEGER NOT NULL UNIQUE, - retries INTEGER NOT NULL DEFAULT 0, - next_run_time TIMESTAMP NOT NULL, - is_resolved BOOLEAN NOT NULL DEFAULT false, - has_given_up BOOLEAN NOT NULL DEFAULT false, - archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX IF NOT EXISTS idx_fi_action_retry_archive_fa_id ON fi_action_retry_archive (fa_id); - -CREATE TABLE IF NOT EXISTS fi_action_error_archive -( - id INTEGER PRIMARY KEY, - fa_id INTEGER NOT NULL, - error TEXT NOT NULL, - message TEXT NOT NULL, - body TEXT, - retry_id INTEGER, - date TIMESTAMP NOT NULL, - archived_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX IF NOT EXISTS idx_fi_action_error_archive_fa_id ON fi_action_error_archive (fa_id); - CREATE TABLE IF NOT EXISTS pair_config ( pair VARCHAR(72) PRIMARY KEY, diff --git a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt index c3701d421..ff7078afd 100644 --- a/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt +++ b/accountant/accountant-ports/accountant-wallet-proxy/src/main/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImpl.kt @@ -11,24 +11,15 @@ import kotlinx.coroutines.reactive.awaitFirstOrNull import org.springframework.beans.factory.annotation.Value import org.springframework.http.MediaType import org.springframework.stereotype.Component -import org.springframework.web.reactive.function.client.WebClientRequestException import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.bodyToMono -import reactor.core.publisher.Mono -import reactor.util.retry.Retry -import java.io.IOException -import java.net.ConnectException import java.math.BigDecimal -import java.time.Duration -import java.util.concurrent.TimeoutException @Component class WalletProxyImpl( private val webClient: WebClient, - @Value("\${app.wallet.url}") private val walletBaseUrl: String, - @Value("\${app.wallet.http.retry.count:2}") private val retryCount: Long = 2, - @Value("\${app.wallet.http.retry.delay-millis:250}") private val retryDelayMillis: Long = 250, - @Value("\${app.wallet.http.timeout-seconds:10}") private val timeoutSeconds: Long = 10 + @Value("\${app.wallet.url}") + private val walletBaseUrl: String ) : WalletProxy { data class TransferBody( @@ -48,15 +39,14 @@ class WalletProxyImpl( transferRef: String?, transferCategory: String ) { - withTransientRetry { - webClient.post() - .uri("$walletBaseUrl/v2/transfer/${amount}_$symbol/from/${senderUuid}_$senderWalletType/to/${receiverUuid}_$receiverWalletType") - .contentType(MediaType.APPLICATION_JSON) - .bodyValue(TransferBody(description, transferRef, transferCategory)) - .retrieve() - .onStatus({ t -> t.isError }, { it.createException() }) - .bodyToMono() - }.awaitFirst() + webClient.post() + .uri("$walletBaseUrl/v2/transfer/${amount}_$symbol/from/${senderUuid}_$senderWalletType/to/${receiverUuid}_$receiverWalletType") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(TransferBody(description, transferRef, transferCategory)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirst() } override suspend fun canFulfil(symbol: String, walletType: WalletType, uuid: String, amount: BigDecimal): Boolean { @@ -66,7 +56,6 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono() - .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirst() .result } @@ -80,7 +69,6 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono() - .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirstOrNull() } @@ -91,27 +79,6 @@ class WalletProxyImpl( .retrieve() .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono>() - .timeout(Duration.ofSeconds(timeoutSeconds)) .awaitFirst() } - - private fun withTransientRetry(request: () -> Mono): Mono { - return request() - .timeout(Duration.ofSeconds(timeoutSeconds)) - .retryWhen( - Retry.backoff(retryCount, Duration.ofMillis(retryDelayMillis)) - .filter { error -> - when { - error is WebClientRequestException -> true - error is TimeoutException -> true - error is ConnectException -> true - error is IOException -> true - error.cause is TimeoutException -> true - error.cause is ConnectException -> true - else -> false - } - } - .onRetryExhaustedThrow { _, signal -> signal.failure() } - ) - } } \ No newline at end of file diff --git a/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt b/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt index 0fe54ce0d..107c2b794 100644 --- a/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt +++ b/accountant/accountant-ports/accountant-wallet-proxy/src/test/kotlin/co/nilin/opex/accountant/ports/walletproxy/proxy/WalletProxyImplTest.kt @@ -22,10 +22,7 @@ class WalletProxyImplTest { private lateinit var mockServer: MockServerClient private val walletProxyImpl = WalletProxyImpl( WebClient.builder().build(), - "http://localhost:8089", - 1, - 10, - 5 + "http://localhost:8089" ) private val objectMapper = ObjectMapper() diff --git a/device-management/pom.xml b/device-management/pom.xml index 0923d7cdd..e57f71f5c 100644 --- a/device-management/pom.xml +++ b/device-management/pom.xml @@ -22,7 +22,7 @@ 2.1.0 3.4.2 2024.0.0 - 1.2.26 + 1.2.27 From 9473b0a2aeac4e6a0d92851e6228673be8de2f7f Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:24:29 +0200 Subject: [PATCH 19/22] Optimize financial action operations * Optimize operations about financial actions * Check the transaction body instead of just the reference to verify duplicate requests --- .../scheduler/FinancialActionsArchiveJob.kt | 19 ++++- .../postgres/dao/FinancialActionRepository.kt | 26 +++++-- .../impl/FinancialActionLoaderImpl.kt | 8 +- .../impl/FinancialActionPersisterImpl.kt | 2 +- .../src/main/resources/schema.sql | 9 +++ .../ports/postgres/FAPersisterImplTest.kt | 29 +++++++ wallet/wallet-app/pom.xml | 5 ++ .../exc/ConcurrentBalanceChangException.kt | 2 +- .../wallet/core/model/PersistedTransaction.kt | 6 ++ .../core/service/TransferManagerImpl.kt | 77 ++++++++++++++++--- .../wallet/core/spi/TransactionManager.kt | 1 + .../core/service/TransferManagerImplTest.kt | 69 +++++++++++++++++ .../postgres/dao/TransactionRepository.kt | 3 + .../postgres/impl/TransactionManagerImpl.kt | 26 ++++++- 14 files changed, 250 insertions(+), 32 deletions(-) create mode 100644 wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/model/PersistedTransaction.kt diff --git a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt index f0d26804b..a6acea9e8 100644 --- a/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt +++ b/accountant/accountant-app/src/main/kotlin/co/nilin/opex/accountant/app/scheduler/FinancialActionsArchiveJob.kt @@ -25,15 +25,26 @@ class FinancialActionsArchiveJob( @Value("\${app.fi-action.archive.batch-size:1000}") private var batchSize: Int = 1000 + @Value("\${app.fi-action.archive.max-batches-per-run:20}") + private var maxBatchesPerRun: Int = 20 + @Scheduled(fixedDelayString = "\${app.fi-action.archive.fixed-delay-ms:300000}", initialDelay = 60000) fun archiveProcessedActions() { - if (!enabled || batchSize <= 0 || retentionDays <= 0) return + if (!enabled || batchSize <= 0 || retentionDays <= 0 || maxBatchesPerRun <= 0) return runBlocking { val before = LocalDateTime.now().minusDays(retentionDays) - val archived = financialActionPersister.archiveProcessedActions(before, batchSize) - if (archived > 0) { - log.info("Archived $archived processed financial actions older than $before") + var totalArchived = 0 + var shouldContinue = true + repeat(maxBatchesPerRun) { + if (!shouldContinue) return@repeat + val archived = financialActionPersister.archiveProcessedActions(before, batchSize) + totalArchived += archived + if (archived < batchSize) shouldContinue = false + } + + if (totalArchived > 0) { + log.info("Archived $totalArchived processed financial actions older than $before") } } } diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt index 16dbeddf4..86b7fb960 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/dao/FinancialActionRepository.kt @@ -9,7 +9,6 @@ import org.springframework.data.repository.query.Param import org.springframework.data.repository.reactive.ReactiveCrudRepository import org.springframework.stereotype.Repository import reactor.core.publisher.Mono -import java.math.BigDecimal import java.time.LocalDateTime @Repository @@ -22,13 +21,23 @@ interface FinancialActionRepository : ReactiveCrudRepository - @Query("select count(1) from fi_actions fi where fi.sender = :uuid and fi.symbol = :symbol and fi.event_type = :eventType and fi.status != :status") - fun countByUuidAndSymbolAndEventTypeAndStatusNot( + @Query( + """ + select exists( + select 1 + from fi_actions fi + where fi.sender = :uuid + and fi.symbol = :symbol + and fi.event_type = :eventType + and fi.status <> 'PROCESSED' + ) + """ + ) + fun existsUnprocessedBySenderAndSymbolAndEventType( @Param("uuid") uuid: String, @Param("symbol") symbol: String, - @Param("eventType") eventType: String, - @Param("status") financialActionStatus: FinancialActionStatus - ): Mono + @Param("eventType") eventType: String + ): Mono @Query("select * from fi_actions fi where status != :status") fun findByStatusNot(@Param("status") status: String, paging: Pageable): Flow @@ -69,6 +78,11 @@ interface FinancialActionRepository : ReactiveCrudRepository 'PROCESSED' + ) order by create_date limit :limit ), diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt index 1fb65e088..c4e281c2c 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionLoaderImpl.kt @@ -15,7 +15,6 @@ import kotlinx.coroutines.reactive.awaitFirstOrElse import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Sort import org.springframework.stereotype.Component -import java.math.BigDecimal import java.time.LocalDateTime @Component @@ -48,12 +47,11 @@ class FinancialActionLoaderImpl( } override suspend fun countUnprocessed(userUuid: String, symbol: String, eventType: String): Long { - return financialActionRepository.countByUuidAndSymbolAndEventTypeAndStatusNot( + return if (financialActionRepository.existsUnprocessedBySenderAndSymbolAndEventType( userUuid, symbol, - eventType, - FinancialActionStatus.PROCESSED - ).awaitFirstOrElse { BigDecimal.ZERO }.toLong() + eventType + ).awaitFirstOrElse { false }) 1L else 0L } override suspend fun loadFinancialAction(id: Long?): FinancialAction? { diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt index 12292c2e6..e5412a6e7 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/kotlin/co/nilin/opex/accountant/ports/postgres/impl/FinancialActionPersisterImpl.kt @@ -103,7 +103,7 @@ class FinancialActionPersisterImpl( faRetryRepository.scheduleNext( id!!, retries + 1, - LocalDateTime.now().plusSeconds(retries * delayMultiplier * delaySeconds), + LocalDateTime.now().plusSeconds((retries + 1L) * delayMultiplier * delaySeconds), giveUp ).awaitSingleOrNull() diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql index 8b7b1b772..994e76d3d 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql +++ b/accountant/accountant-ports/accountant-persister-postgres/src/main/resources/schema.sql @@ -51,6 +51,15 @@ CREATE INDEX IF NOT EXISTS idx_fi_actions_status ON fi_actions (status); CREATE INDEX IF NOT EXISTS idx_fi_actions_pointer ON fi_actions (pointer); CREATE INDEX IF NOT EXISTS idx_fi_actions_status_create_date ON fi_actions (status, create_date); CREATE INDEX IF NOT EXISTS idx_fi_actions_parent_status ON fi_actions (parent_id, status); +CREATE INDEX IF NOT EXISTS idx_fi_actions_unprocessed_lookup + ON fi_actions (sender, symbol, event_type) + WHERE status <> 'PROCESSED'; +CREATE INDEX IF NOT EXISTS idx_fi_actions_archive_candidates + ON fi_actions (create_date, id) + WHERE status = 'PROCESSED'; +CREATE INDEX IF NOT EXISTS idx_fi_actions_unprocessed_children_by_parent + ON fi_actions (parent_id) + WHERE status <> 'PROCESSED'; ALTER TABLE fi_actions ADD COLUMN IF NOT EXISTS category_name VARCHAR(36); diff --git a/accountant/accountant-ports/accountant-persister-postgres/src/test/kotlin/co/nilin/opex/accountant/ports/postgres/FAPersisterImplTest.kt b/accountant/accountant-ports/accountant-persister-postgres/src/test/kotlin/co/nilin/opex/accountant/ports/postgres/FAPersisterImplTest.kt index 38c794ffd..ea63bfebf 100644 --- a/accountant/accountant-ports/accountant-persister-postgres/src/test/kotlin/co/nilin/opex/accountant/ports/postgres/FAPersisterImplTest.kt +++ b/accountant/accountant-ports/accountant-persister-postgres/src/test/kotlin/co/nilin/opex/accountant/ports/postgres/FAPersisterImplTest.kt @@ -9,10 +9,14 @@ import co.nilin.opex.accountant.ports.postgres.model.FinancialActionModel import io.mockk.coEvery import io.mockk.coVerify import io.mockk.mockk +import io.mockk.slot +import co.nilin.opex.accountant.ports.postgres.model.FinancialActionRetryModel import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import java.time.LocalDateTime @Suppress("ReactiveStreamsUnusedPublisher") class FAPersisterImplTest { @@ -48,4 +52,29 @@ class FAPersisterImplTest { } } + @Test + fun givenRetryableAction_whenUpdateWithError_thenScheduleUsesBackoffDelay(): Unit = runBlocking { + val retryModel = FinancialActionRetryModel( + faId = Valid.fa.id!!, + nextRunTime = LocalDateTime.now(), + retries = 0, + isResolved = false, + hasGivenUp = false, + id = 10 + ) + val nextRunSlot = slot() + + coEvery { faRetryRepository.findByFaId(Valid.fa.id!!) } returns Mono.just(retryModel) + coEvery { faRetryRepository.scheduleNext(eq(10), eq(1), capture(nextRunSlot), eq(false)) } returns Mono.empty() + coEvery { financialActionRepository.updateStatus(eq(Valid.fa.id!!), eq(FinancialActionStatus.RETRYING)) } returns Mono.empty() + coEvery { faErrorRepository.save(any()) } returns Mono.empty() + + val before = LocalDateTime.now() + faPersister.updateWithError(Valid.fa, "ERR", "message", null) + + coVerify(exactly = 1) { faRetryRepository.scheduleNext(eq(10), eq(1), any(), eq(false)) } + assertTrue(nextRunSlot.isCaptured) + assertTrue(nextRunSlot.captured.isAfter(before.plusSeconds(10))) + } + } \ No newline at end of file diff --git a/wallet/wallet-app/pom.xml b/wallet/wallet-app/pom.xml index 9e1e23288..13dd14714 100644 --- a/wallet/wallet-app/pom.xml +++ b/wallet/wallet-app/pom.xml @@ -251,6 +251,11 @@ 5.4.0 test + + com.zaxxer + HikariCP + test + diff --git a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/exc/ConcurrentBalanceChangException.kt b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/exc/ConcurrentBalanceChangException.kt index edf52fad0..c8ab6b465 100644 --- a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/exc/ConcurrentBalanceChangException.kt +++ b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/exc/ConcurrentBalanceChangException.kt @@ -1,3 +1,3 @@ package co.nilin.opex.wallet.core.exc -class ConcurrentBalanceChangException(override val message: String?) : Exception() \ No newline at end of file +class ConcurrentBalanceChangException(override val message: String?) : RuntimeException() \ No newline at end of file diff --git a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/model/PersistedTransaction.kt b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/model/PersistedTransaction.kt new file mode 100644 index 000000000..439062dec --- /dev/null +++ b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/model/PersistedTransaction.kt @@ -0,0 +1,6 @@ +package co.nilin.opex.wallet.core.model + +data class PersistedTransaction( + val id: Long, + val transaction: Transaction +) diff --git a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImpl.kt b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImpl.kt index b03fb97ed..8fc6cd9dc 100644 --- a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImpl.kt +++ b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImpl.kt @@ -7,6 +7,7 @@ import co.nilin.opex.wallet.core.inout.TransferResultDetailed import co.nilin.opex.wallet.core.model.* import co.nilin.opex.wallet.core.spi.* import org.slf4j.LoggerFactory +import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -25,6 +26,8 @@ class TransferManagerImpl( @Transactional override suspend fun transfer(transferCommand: TransferCommand): TransferResultDetailed { + resolveIdempotentTransfer(transferCommand)?.let { return it } + //pre transfer hook (dispatch pre transfer event) val srcWallet = transferCommand.sourceWallet val srcWalletOwner = srcWallet.owner @@ -53,20 +56,26 @@ class TransferManagerImpl( if (!walletManager.isDepositAllowed(destWallet, amountToTransfer)) throw OpexError.DepositLimitExceeded.exception() + val tx = try { + transactionManager.save( + Transaction( + srcWallet, + destWallet, + transferCommand.amount.amount, + amountToTransfer, + transferCommand.description, + transferCommand.transferRef, + transferCommand.transferCategory, + LocalDateTime.now() + ) + ) + } catch (e: DuplicateKeyException) { + resolveIdempotentTransfer(transferCommand)?.let { return it } + throw e + } + walletManager.decreaseBalance(srcWallet, transferCommand.amount.amount) walletManager.increaseBalance(destWallet, amountToTransfer) - val tx = transactionManager.save( - Transaction( - srcWallet, - destWallet, - transferCommand.amount.amount, - amountToTransfer, - transferCommand.description, - transferCommand.transferRef, - transferCommand.transferCategory, - LocalDateTime.now() - ) - ) //TODO make tx long by default createUserTX(transferCommand, tx) @@ -93,6 +102,50 @@ class TransferManagerImpl( ) } + private suspend fun resolveIdempotentTransfer(transferCommand: TransferCommand): TransferResultDetailed? { + val transferRef = transferCommand.transferRef ?: return null + val persistedTransaction = transactionManager.findTransactionByTransferRef(transferRef) ?: return null + val existingTxId = persistedTransaction.id + val existingTransaction = persistedTransaction.transaction + + if (!matchesIdempotentTransfer(transferCommand, existingTransaction)) { + throw OpexError.BadRequest.exception("transferRef=$transferRef already exists with different parameters") + } + + logger.info("Idempotent transfer hit for transferRef={}", transferRef) + return buildIdempotentResult(existingTransaction, existingTxId) + } + + private fun matchesIdempotentTransfer(transferCommand: TransferCommand, existingTransaction: Transaction): Boolean { + return transferCommand.sourceWallet.id == existingTransaction.sourceWallet.id && + transferCommand.destWallet.id == existingTransaction.destWallet.id && + transferCommand.amount == Amount(existingTransaction.sourceWallet.currency, existingTransaction.sourceAmount) && + transferCommand.destAmount == Amount(existingTransaction.destWallet.currency, existingTransaction.destAmount) && + transferCommand.transferCategory == existingTransaction.transferCategory && + transferCommand.description == existingTransaction.description + } + + private fun buildIdempotentResult(existingTransaction: Transaction, existingTxId: Long): TransferResultDetailed { + val srcWallet = existingTransaction.sourceWallet + val destWallet = existingTransaction.destWallet + return TransferResultDetailed( + TransferResult( + Date().time, + srcWallet.owner.uuid, + srcWallet.type, + srcWallet.balance, + srcWallet.balance, + Amount(srcWallet.currency, existingTransaction.sourceAmount), + destWallet.owner.uuid, + destWallet.type, + Amount(destWallet.currency, existingTransaction.destAmount), + srcWallet.id, + destWallet.id, + ), + existingTxId.toString() + ) + } + private suspend fun createUserTX(command: TransferCommand, txId: Long) { val currency = command.amount.currency.symbol val amount = command.amount.amount diff --git a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/spi/TransactionManager.kt b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/spi/TransactionManager.kt index 9f646d823..d8fff39c0 100644 --- a/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/spi/TransactionManager.kt +++ b/wallet/wallet-core/src/main/kotlin/co/nilin/opex/wallet/core/spi/TransactionManager.kt @@ -6,6 +6,7 @@ import java.time.LocalDateTime interface TransactionManager { suspend fun save(transaction: Transaction): Long + suspend fun findTransactionByTransferRef(transferRef: String): PersistedTransaction? suspend fun findDepositTransactions( uuid: String, diff --git a/wallet/wallet-core/src/test/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImplTest.kt b/wallet/wallet-core/src/test/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImplTest.kt index 0fc406bfa..08320d369 100644 --- a/wallet/wallet-core/src/test/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImplTest.kt +++ b/wallet/wallet-core/src/test/kotlin/co/nilin/opex/wallet/core/service/TransferManagerImplTest.kt @@ -1,15 +1,20 @@ package co.nilin.opex.wallet.core.service +import co.nilin.opex.common.OpexError import co.nilin.opex.wallet.core.model.Amount +import co.nilin.opex.wallet.core.model.Transaction import co.nilin.opex.wallet.core.service.sample.VALID import co.nilin.opex.wallet.core.spi.* import io.mockk.MockKException import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.mockk import kotlinx.coroutines.runBlocking import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test +import java.math.BigDecimal private class TransferManagerImplTest { private val walletOwnerManager: WalletOwnerManager = mockk() @@ -169,4 +174,68 @@ private class TransferManagerImplTest { } }.isNotInstanceOf(MockKException::class.java) } + + @Test + fun givenExistingTransferRef_whenTransfer_thenReturnIdempotentSuccessWithoutBalanceChanges(): Unit = runBlocking { + val command = VALID.TRANSFER_COMMAND.copy(transferRef = "accountant:fiActions:abc") + coEvery { transactionManager.findTransactionByTransferRef(eq(command.transferRef!!)) } returns co.nilin.opex.wallet.core.model.PersistedTransaction( + 100L, + Transaction( + VALID.SOURCE_WALLET, + VALID.DEST_WALLET, + command.amount.amount, + command.destAmount.amount, + command.description, + command.transferRef, + command.transferCategory, + java.time.LocalDateTime.now() + ) + ) + + val result = transferManager.transfer(command) + + assertThat(result.tx).isEqualTo("100") + assertThat(result.transferResult.sourceUuid).isEqualTo(command.sourceWallet.owner.uuid) + assertThat(result.transferResult.destUuid).isEqualTo(command.destWallet.owner.uuid) + + coVerify(exactly = 0) { walletManager.decreaseBalance(any(), any()) } + coVerify(exactly = 0) { walletManager.increaseBalance(any(), any()) } + coVerify(exactly = 0) { transactionManager.save(any()) } + coVerify(exactly = 0) { walletListener.onDeposit(any(), any(), any(), any(), any()) } + coVerify(exactly = 0) { walletListener.onWithdraw(any(), any(), any(), any()) } + } + + @Test + fun givenExistingTransferRefWithDifferentParams_whenTransfer_thenThrowBadRequest(): Unit = runBlocking { + val command = VALID.TRANSFER_COMMAND.copy( + transferRef = "accountant:fiActions:abc", + destWallet = VALID.DEST_WALLET.copy(id = 999L), + amount = Amount(VALID.CURRENCY, BigDecimal("0.75")), + destAmount = Amount(VALID.CURRENCY, BigDecimal("0.75")) + ) + coEvery { transactionManager.findTransactionByTransferRef(eq(command.transferRef!!)) } returns co.nilin.opex.wallet.core.model.PersistedTransaction( + 100L, + Transaction( + VALID.SOURCE_WALLET, + VALID.DEST_WALLET, + VALID.TRANSFER_COMMAND.amount.amount, + VALID.TRANSFER_COMMAND.destAmount.amount, + VALID.TRANSFER_COMMAND.description, + VALID.TRANSFER_COMMAND.transferRef, + VALID.TRANSFER_COMMAND.transferCategory, + java.time.LocalDateTime.now() + ) + ) + + val ex = Assertions.assertThrows(co.nilin.opex.utility.error.data.OpexException::class.java) { + runBlocking { + transferManager.transfer(command) + } + } + + assertThat(ex.error).isEqualTo(OpexError.BadRequest) + coVerify(exactly = 0) { walletManager.decreaseBalance(any(), any()) } + coVerify(exactly = 0) { walletManager.increaseBalance(any(), any()) } + coVerify(exactly = 0) { transactionManager.save(any()) } + } } diff --git a/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/dao/TransactionRepository.kt b/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/dao/TransactionRepository.kt index f47ca1312..0eba0f408 100644 --- a/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/dao/TransactionRepository.kt +++ b/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/dao/TransactionRepository.kt @@ -16,6 +16,9 @@ import java.time.LocalDateTime @Repository interface TransactionRepository : ReactiveCrudRepository { + @Query("select * from transaction where transfer_ref = :transferRef limit 1") + fun findByTransferRef(transferRef: String): Mono + @Query( """ SELECT count(1) cnt, COALESCE(sum(source_amount), 0) total diff --git a/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/impl/TransactionManagerImpl.kt b/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/impl/TransactionManagerImpl.kt index 7e7ec45e0..d60f33884 100644 --- a/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/impl/TransactionManagerImpl.kt +++ b/wallet/wallet-ports/wallet-persister-postgres/src/main/kotlin/co/nilin/opex/wallet/ports/postgres/impl/TransactionManagerImpl.kt @@ -2,12 +2,14 @@ package co.nilin.opex.wallet.ports.postgres.impl import co.nilin.opex.wallet.core.model.* import co.nilin.opex.wallet.core.spi.TransactionManager +import co.nilin.opex.wallet.core.spi.WalletManager import co.nilin.opex.wallet.ports.postgres.dao.CurrencyRepositoryV2 import co.nilin.opex.wallet.ports.postgres.dao.TransactionRepository import co.nilin.opex.wallet.ports.postgres.model.TransactionModel import com.fasterxml.jackson.databind.ObjectMapper import kotlinx.coroutines.reactive.awaitFirstOrElse import kotlinx.coroutines.reactive.awaitSingle +import kotlinx.coroutines.reactor.awaitSingleOrNull import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.time.LocalDateTime @@ -17,6 +19,7 @@ import java.time.ZoneId class TransactionManagerImpl( private val transactionRepository: TransactionRepository, private val currencyRepositoryV2: CurrencyRepositoryV2, + private val walletManager: WalletManager, private val objectMapper: ObjectMapper ) : TransactionManager { private val logger = LoggerFactory.getLogger(TransactionManagerImpl::class.java) @@ -36,6 +39,26 @@ class TransactionManagerImpl( ).awaitSingle().id!! } + override suspend fun findTransactionByTransferRef(transferRef: String): PersistedTransaction? { + val transaction = transactionRepository.findByTransferRef(transferRef).awaitSingleOrNull() ?: return null + val sourceWallet = walletManager.findWalletById(transaction.sourceWallet) ?: return null + val destWallet = walletManager.findWalletById(transaction.destWallet) ?: return null + + return PersistedTransaction( + transaction.id!!, + Transaction( + sourceWallet, + destWallet, + transaction.sourceAmount, + transaction.destAmount, + transaction.description, + transaction.transferRef, + transaction.transferCategory, + transaction.transactionDate + ) + ) + } + override suspend fun findDepositTransactions( uuid: String, @@ -148,6 +171,3 @@ class TransactionManagerImpl( .collectList().awaitFirstOrElse { emptyList() } } } - - - From 7050bdf4ec8627b374c35f6626527761d54c86d5 Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:56:20 +0330 Subject: [PATCH 20/22] Implement two-factor authentication (#720) --- .../co/nilin/opex/api/core/inout/OTPType.kt | 2 +- .../nilin/opex/api/core/inout/PairCategory.kt | 7 + .../opex/api/core/inout/PairInfoResponse.kt | 3 + .../nilin/opex/api/core/inout/PairSetting.kt | 4 + .../co/nilin/opex/api/core/inout/TOTP.kt | 28 ++ .../co/nilin/opex/api/core/inout/TwoFactor.kt | 15 + .../co/nilin/opex/api/core/spi/AuthProxy.kt | 14 + .../ports/binance/config/SecurityConfig.kt | 1 + .../ports/opex/controller/MarketController.kt | 12 +- .../controller/UserTwoFactorController.kt | 259 ++++++++++++++++++ .../api/ports/proxy/impl/AuthProxyImpl.kt | 101 +++++++ .../auth/controller/PublicUserController.kt | 42 ++- .../controller/UserTwoFactorController.kt | 239 ++++++++++++++++ .../kotlin/co/nilin/opex/auth/model/OTP.kt | 2 +- .../kotlin/co/nilin/opex/auth/model/TOTP.kt | 28 ++ .../co/nilin/opex/auth/model/TwoFactor.kt | 15 + .../co/nilin/opex/auth/model/UserRegister.kt | 2 - .../co/nilin/opex/auth/proxy/KeycloakProxy.kt | 58 +++- .../co/nilin/opex/auth/proxy/OTPProxy.kt | 54 +++- .../auth/service/ForgetPasswordService.kt | 12 +- .../nilin/opex/auth/service/LoginService.kt | 143 +++++++--- .../opex/auth/service/RegisterService.kt | 25 +- .../auth/service/TwoFactorConfigService.kt | 185 +++++++++++++ .../src/main/resources/application.yml | 1 + .../kotlin/co/nilin/opex/common/OpexError.kt | 1 + docker-compose.yml | 1 + .../opex/otp/app/controller/TOTPController.kt | 7 +- .../opex/otp/app/model/TOTPQueryResponse.kt | 1 + .../nilin/opex/otp/app/service/TOTPService.kt | 16 +- 29 files changed, 1194 insertions(+), 84 deletions(-) create mode 100644 api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt create mode 100644 api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TOTP.kt create mode 100644 api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TwoFactor.kt create mode 100644 api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/UserTwoFactorController.kt create mode 100644 auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/UserTwoFactorController.kt create mode 100644 auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TOTP.kt create mode 100644 auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TwoFactor.kt create mode 100644 auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/TwoFactorConfigService.kt diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/OTPType.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/OTPType.kt index 45d19c8e3..2fce50306 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/OTPType.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/OTPType.kt @@ -2,5 +2,5 @@ package co.nilin.opex.api.core.inout enum class OTPType { - SMS, EMAIL, + SMS, EMAIL, TOTP, NONE } \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt new file mode 100644 index 000000000..b6b92ce16 --- /dev/null +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairCategory.kt @@ -0,0 +1,7 @@ +package co.nilin.opex.api.core.inout + +enum class PairCategory { + REAL_ASSET_TOKEN, + FIAT, + CRYPTO +} \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt index d6c00fe0e..1fed02103 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairInfoResponse.kt @@ -10,4 +10,7 @@ data class PairInfoResponse( val minOrder : BigDecimal, val maxOrder : BigDecimal, val orderTypes : String, + val internalChart: Boolean, + val globalChart: Boolean, + val categories: List = emptyList() ) \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt index b3d7d5946..28f3b951d 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/PairSetting.kt @@ -10,4 +10,8 @@ class PairSetting( val maxOrder : BigDecimal, val orderTypes : String, val updateDate: LocalDateTime? = null, + val internalChart: Boolean, + val globalChart: Boolean, + val categories: List = emptyList() + ) \ No newline at end of file diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TOTP.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TOTP.kt new file mode 100644 index 000000000..8a51cd4ab --- /dev/null +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TOTP.kt @@ -0,0 +1,28 @@ +package co.nilin.opex.api.core.inout + +data class SetupTOTPRequest( + val userId: String, + val label: String? +) + +data class SetupTOTPResponse( + val uri: String +) + +data class VerifyTOTPRequest( + val userId: String, + val code: String +) + +data class VerifyTOTPResponse(val result: Boolean) + +data class TOTPQueryResponse( + val userId: String, + val isEnabled: Boolean, + val isActivated: Boolean, + val uri : String +) + +data class TOTPCode( + val code: String +) diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TwoFactor.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TwoFactor.kt new file mode 100644 index 000000000..b3f057e2a --- /dev/null +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/TwoFactor.kt @@ -0,0 +1,15 @@ +package co.nilin.opex.api.core.inout + +import com.fasterxml.jackson.annotation.JsonInclude + +data class TwoFactorRequest( + val method: OTPType, +) + +data class ConfirmTwoFactorRequest( + val method: OTPType, + val otp: String, +) + +@JsonInclude(JsonInclude.Include.NON_NULL) +data class TwoFactorResponse(val otp: String?, val otpReceiver: OTPReceiver?) diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/spi/AuthProxy.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/spi/AuthProxy.kt index fb1d6c3f1..f273b4824 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/spi/AuthProxy.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/spi/AuthProxy.kt @@ -1,5 +1,11 @@ package co.nilin.opex.api.core.spi +import co.nilin.opex.api.core.inout.ConfirmTwoFactorRequest +import co.nilin.opex.api.core.inout.OTPType +import co.nilin.opex.api.core.inout.SetupTOTPResponse +import co.nilin.opex.api.core.inout.TOTPCode +import co.nilin.opex.api.core.inout.TwoFactorRequest +import co.nilin.opex.api.core.inout.TwoFactorResponse import co.nilin.opex.api.core.inout.auth.* interface AuthProxy { @@ -22,4 +28,12 @@ interface AuthProxy { suspend fun logoutOthers(token: String) suspend fun logoutAll(token: String) + suspend fun getTwoFactorConfig(token: String): OTPType + suspend fun requestEnableTwoFactor(request: TwoFactorRequest, token: String): TwoFactorResponse + suspend fun confirmEnableTwoFactor(request: ConfirmTwoFactorRequest, token: String): OTPVerifyResponse + suspend fun requestDisableTwoFactor(request: TwoFactorRequest, token: String): TwoFactorResponse + suspend fun confirmDisableTwoFactor(request: ConfirmTwoFactorRequest, token: String): OTPVerifyResponse + suspend fun setupTOTP(token: String): SetupTOTPResponse + suspend fun verifyTOTPSetup(request: TOTPCode, token: String) + } \ No newline at end of file diff --git a/api/api-ports/api-binance-rest/src/main/kotlin/co/nilin/opex/api/ports/binance/config/SecurityConfig.kt b/api/api-ports/api-binance-rest/src/main/kotlin/co/nilin/opex/api/ports/binance/config/SecurityConfig.kt index 4d005fdb5..8203fff40 100644 --- a/api/api-ports/api-binance-rest/src/main/kotlin/co/nilin/opex/api/ports/binance/config/SecurityConfig.kt +++ b/api/api-ports/api-binance-rest/src/main/kotlin/co/nilin/opex/api/ports/binance/config/SecurityConfig.kt @@ -106,6 +106,7 @@ class SecurityConfig( // Opex endpoints .pathMatchers("/opex/v1/oauth/protocol/openid-connect/**").permitAll() .pathMatchers("/opex/v1/oauth.***").permitAll() + .pathMatchers("/opex/v1/user/2fa/**").authenticated() .pathMatchers("/opex/v1/user/public/**").permitAll() .pathMatchers("/opex/v1/user/update/**").permitAll() .pathMatchers("/v1/deposit/webhook").permitAll() diff --git a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt index 290c55827..e309a0a4e 100644 --- a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt +++ b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/MarketController.kt @@ -18,11 +18,7 @@ import io.swagger.v3.oas.annotations.tags.Tag import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.springframework.beans.factory.annotation.Value -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.RestController +import org.springframework.web.bind.annotation.* import java.math.BigDecimal import java.time.ZoneId @@ -104,7 +100,11 @@ Response body: isAvailable = isAvailable, minOrder = minOrder, maxOrder = maxOrder, - orderTypes = orderTypes + orderTypes = orderTypes, + internalChart = internalChart, + globalChart = globalChart, + categories = categories + ) } } diff --git a/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/UserTwoFactorController.kt b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/UserTwoFactorController.kt new file mode 100644 index 000000000..34b7e5528 --- /dev/null +++ b/api/api-ports/api-opex-rest/src/main/kotlin/co/nilin/opex/api/ports/opex/controller/UserTwoFactorController.kt @@ -0,0 +1,259 @@ +package co.nilin.opex.api.ports.opex.controller + +import co.nilin.opex.api.core.inout.ConfirmTwoFactorRequest +import co.nilin.opex.api.core.inout.OTPType +import co.nilin.opex.api.core.inout.SetupTOTPResponse +import co.nilin.opex.api.core.inout.TOTPCode +import co.nilin.opex.api.core.inout.TwoFactorRequest +import co.nilin.opex.api.core.inout.TwoFactorResponse +import co.nilin.opex.api.core.inout.auth.* +import co.nilin.opex.api.core.spi.AuthProxy +import co.nilin.opex.api.ports.opex.util.jwtAuthentication +import co.nilin.opex.api.ports.opex.util.tokenValue +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.media.Content +import io.swagger.v3.oas.annotations.media.Schema +import io.swagger.v3.oas.annotations.responses.ApiResponse +import io.swagger.v3.oas.annotations.security.SecurityRequirement +import io.swagger.v3.oas.annotations.tags.Tag +import org.springframework.http.ResponseEntity +import org.springframework.security.core.annotation.CurrentSecurityContext +import org.springframework.security.core.context.SecurityContext +import org.springframework.web.bind.annotation.* + +@RestController +@RequestMapping("/opex/v1/user/2fa") +@Tag( + name = "User Two-Factor Configuration", + description = "Endpoints for managing user two-factor authentication (2FA) settings and TOTP setup." +) +@SecurityRequirement(name = "bearerAuth") +class UserTwoFactorController(private val authProxy: AuthProxy) { + + @GetMapping + @Operation( + summary = "Get current two-factor authentication configuration", + description = """GET /opex/v1/user/2fa. +Security: Bearer token is required. + +Behavior: Retrieves the currently active two-factor authentication (2FA) method for the authenticated user. +Possible return values: NONE, EMAIL, SMS, TOTP.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor configuration retrieved successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = OTPType::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun getTwoFactorConfig( + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.getTwoFactorConfig(securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/enable/request") + @Operation( + summary = "Request enabling two-factor authentication", + description = """POST /opex/v1/user/2fa/enable/request. +Security: Bearer token is required. + +Behavior: Starts the two-factor authentication enable flow for the authenticated user. +Allowed values: +- method: EMAIL, SMS, TOTP""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor enable request created successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = TwoFactorResponse::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun requestEnableTwoFactor( + @RequestBody request: TwoFactorRequest, + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.requestEnableTwoFactor(request, securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/enable/confirm") + @Operation( + summary = "Confirm enabling two-factor authentication", + description = """POST /opex/v1/user/2fa/enable/confirm. +Security: Bearer token is required. + +Behavior: Confirms and activates the two-factor authentication enable flow for the authenticated user.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor authentication enabled successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = OTPVerifyResponse::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun confirmEnableTwoFactor( + @RequestBody request: ConfirmTwoFactorRequest, + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.confirmEnableTwoFactor(request, securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/disable/request") + @Operation( + summary = "Request disabling two-factor authentication", + description = """POST /opex/v1/user/2fa/disable/request. +Security: Bearer token is required. + +Behavior: Starts the two-factor authentication disable flow for the authenticated user.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor disable request created successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = TwoFactorResponse::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun requestDisableTwoFactor( + @RequestBody request: TwoFactorRequest, + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.requestDisableTwoFactor(request, securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/disable/confirm") + @Operation( + summary = "Confirm disabling two-factor authentication", + description = """POST /opex/v1/user/2fa/disable/confirm. +Security: Bearer token is required. + +Behavior: Confirms and disables two-factor authentication for the authenticated user.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor authentication disabled successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = OTPVerifyResponse::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun confirmDisableTwoFactor( + @RequestBody request: ConfirmTwoFactorRequest, + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.confirmDisableTwoFactor(request, securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/totp/setup") + @Operation( + summary = "Setup TOTP (Authenticator App)", + description = """POST /opex/v1/user/2fa/totp/setup. +Security: Bearer token is required. + +Behavior: Generates secret key and setup URL (otpauth://...) for setting up Authenticator app (e.g., Google Authenticator).""", + responses = [ + ApiResponse( + responseCode = "200", + description = "TOTP setup credentials generated successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = SetupTOTPResponse::class) + ) + ] + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun setupTOTP( + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = authProxy.setupTOTP(securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok(response) + } + + @PostMapping("/totp/verify") + @Operation( + summary = "Verify TOTP setup code", + description = """POST /opex/v1/user/2fa/totp/verify. +Security: Bearer token is required. + +Behavior: Verifies the generated TOTP code during the initial authenticator setup phase.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "TOTP setup code verified successfully." + ), + ApiResponse( + responseCode = "401", + description = "Unauthorized. Bearer token is missing, invalid, or expired.", + content = [Content()] + ) + ] + ) + suspend fun verifyTOTPSetup( + @RequestBody request: TOTPCode, + @Parameter(hidden = true) @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + authProxy.verifyTOTPSetup(request, securityContext.jwtAuthentication().tokenValue()) + return ResponseEntity.ok().build() + } +} \ No newline at end of file diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt index 8f702d2a4..358a34dd4 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/AuthProxyImpl.kt @@ -1,5 +1,11 @@ package co.nilin.opex.api.ports.proxy.impl +import co.nilin.opex.api.core.inout.ConfirmTwoFactorRequest +import co.nilin.opex.api.core.inout.OTPType +import co.nilin.opex.api.core.inout.SetupTOTPResponse +import co.nilin.opex.api.core.inout.TOTPCode +import co.nilin.opex.api.core.inout.TwoFactorRequest +import co.nilin.opex.api.core.inout.TwoFactorResponse import co.nilin.opex.api.core.inout.auth.* import co.nilin.opex.api.core.spi.AuthProxy import co.nilin.opex.common.OpexError @@ -223,4 +229,99 @@ class AuthProxyImpl(@Qualifier("generalWebClient") private val webClient: WebCli } .awaitBodilessEntity() } + + override suspend fun getTwoFactorConfig(token: String): OTPType { + return webClient.get() + .uri("$baseUrl/v1/user/2fa") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to get 2fa config") } + } + + override suspend fun requestEnableTwoFactor( + request: TwoFactorRequest, + token: String + ): TwoFactorResponse { + return webClient.post() + .uri("$baseUrl/v1/user/2fa/enable/request") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .body(Mono.just(request)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to request enable 2fa") } + } + + override suspend fun confirmEnableTwoFactor( + request: ConfirmTwoFactorRequest, + token: String + ): OTPVerifyResponse { + return webClient.post() + .uri("$baseUrl/v1/user/2fa/enable/confirm") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .body(Mono.just(request)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to confirm enable 2fa") } + } + + override suspend fun requestDisableTwoFactor( + request: TwoFactorRequest, + token: String + ): TwoFactorResponse { + return webClient.post() + .uri("$baseUrl/v1/user/2fa/disable/request") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .body(Mono.just(request)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to request disable 2fa") } + } + + override suspend fun confirmDisableTwoFactor( + request: ConfirmTwoFactorRequest, + token: String + ): OTPVerifyResponse { + return webClient.post() + .uri("$baseUrl/v1/user/2fa/disable/confirm") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .body(Mono.just(request)) + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to confirm disable 2fa") } + } + + override suspend fun setupTOTP(token: String): SetupTOTPResponse { + return webClient.post() + .uri("$baseUrl/v1/user/2fa/totp/setup") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .retrieve() + .onStatus({ t -> t.isError }, { it.createException() }) + .bodyToMono() + .awaitFirstOrElse { throw OpexError.BadRequest.exception("Failed to setup TOTP") } + } + + override suspend fun verifyTOTPSetup(request: TOTPCode, token: String) { + webClient.post() + .uri("$baseUrl/v1/user/2fa/totp/verify") + .accept(MediaType.APPLICATION_JSON) + .header(HttpHeaders.AUTHORIZATION, "Bearer $token") + .body(Mono.just(request)) + .retrieve() + .onStatus({ it.isError }) { response -> + response.createException() + } + .awaitBodilessEntity() + } } \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/PublicUserController.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/PublicUserController.kt index 7b48faeaa..a7fd2cffb 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/PublicUserController.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/PublicUserController.kt @@ -49,6 +49,26 @@ Allowed values: return ResponseEntity.ok().body(otpResponse) } + @PostMapping("/register/resend-otp") + @Operation( + summary = "Resend registration OTP", + description = """POST /v1/user/public/register/resend-otp. +Security: Public endpoint. No Bearer token is required. + +Behavior: Resends the registration OTP.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Successful response.", + content = [Content(mediaType = "application/json", schema = Schema(type = "object"))] + ) + ] + ) + suspend fun resendRegistrationOtp(@Valid @RequestBody request: ResendOtpRequest): ResponseEntity { + val otpResponse = registerService.resendRegistrationOtp(request) + return ResponseEntity.ok().body(otpResponse) + } + @PostMapping("/register/verify") @Operation( summary = "Verify registration OTP", @@ -140,6 +160,26 @@ Allowed values: return ResponseEntity.ok().body(otpResponse) } + @PostMapping("/forget/resend-otp") + @Operation( + summary = "Resend forgot-password OTP", + description = """POST /v1/user/public/forget/resend-otp. +Security: Public endpoint. No Bearer token is required. + +Behavior: Resends the forgot-password OTP.""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Successful response.", + content = [Content(mediaType = "application/json", schema = Schema(type = "object"))] + ) + ] + ) + suspend fun resendForgetOtp(@Valid @RequestBody request: ResendOtpRequest): ResponseEntity { + val otpResponse = forgetPasswordService.resendForgetOtp(request) + return ResponseEntity.ok().body(otpResponse) + } + @PostMapping("/forget/verify") @Operation( summary = "Verify forgot-password OTP", @@ -184,4 +224,4 @@ Response body: No response body.""", forgetPasswordService.confirmForget(request) return ResponseEntity.ok().build() } -} +} \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/UserTwoFactorController.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/UserTwoFactorController.kt new file mode 100644 index 000000000..653dbf726 --- /dev/null +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/controller/UserTwoFactorController.kt @@ -0,0 +1,239 @@ +package co.nilin.opex.auth.controller + +import co.nilin.opex.auth.model.* +import co.nilin.opex.auth.service.TwoFactorConfigService +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.media.Content +import io.swagger.v3.oas.annotations.media.Schema +import io.swagger.v3.oas.annotations.responses.ApiResponse +import io.swagger.v3.oas.annotations.tags.Tag +import org.springframework.http.ResponseEntity +import org.springframework.security.core.annotation.CurrentSecurityContext +import org.springframework.security.core.context.SecurityContext +import org.springframework.web.bind.annotation.* + +@RestController +@RequestMapping("/v1/user/2fa") +@Tag( + name = "User Two-Factor Configuration", + description = "Endpoints for managing user two-factor authentication (2FA) settings and TOTP setup." +) +class UserTwoFactorController(private val twoFactorConfigService: TwoFactorConfigService) { + + + @GetMapping + @Operation( + summary = "Get current two-factor authentication configuration", + description = """ +GET /v1/2fa + +Security: Bearer token is required. + +Behavior: +Retrieves the currently active two-factor authentication (2FA) method for the authenticated user. + +Possible return values: +- NONE: Two-factor authentication is disabled. +- EMAIL: 2FA via Email OTP is active. +- SMS: 2FA via SMS OTP is active. +- TOTP: 2FA via Authenticator App (Time-based OTP) is active. +""", + responses = [ + ApiResponse( + responseCode = "200", + description = "Two-factor configuration retrieved successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = OTPType::class) + ) + ] + ) + ] + ) + suspend fun getTwoFactorConfig(@CurrentSecurityContext securityContext: SecurityContext): OTPType { + return twoFactorConfigService.getTwoFactorConfig(securityContext.authentication.name) + } + + @PostMapping("/enable/request") + @Operation( + summary = "Request enabling two-factor authentication", description = """ +POST /v1/2fa/enable/request. + +Security: Bearer token is required. + +Behavior: +Starts the two-factor authentication enable flow for the authenticated user. + +Allowed values: +- method: EMAIL, SMS, TOTP + +Response: +- EMAIL/SMS: Returns the OTP receiver information. An OTP is sent to the selected receiver. +- TOTP: Returns the TOTP setup URI (otpauth://...) to be used for QR code generation or manual setup. +""", responses = [ApiResponse( + responseCode = "200", description = "Two-factor enable request created successfully.", content = [Content( + mediaType = "application/json", schema = Schema(implementation = TwoFactorResponse::class) + )] + )] + ) + suspend fun requestEnableTwoFactor( + @RequestBody request: TwoFactorRequest, @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = twoFactorConfigService.requestEnableTwoFactor( + request.method, securityContext.authentication.name + ) + return ResponseEntity.ok(response) + } + + @PostMapping("/enable/confirm") + @Operation( + summary = "Confirm enabling two-factor authentication", description = """ +POST /v1/2fa/enable/confirm. + +Security: Bearer token is required. + +Behavior: +Confirm the two-factor authentication enable flow for the authenticated user. + +Allowed values: +- method: EMAIL, SMS, TOTP +- otp : String + +Response: +- Returns the otp result. +""", responses = [ApiResponse( + responseCode = "200", description = "Two-factor authentication enabled successfully.", content = [Content( + mediaType = "application/json", schema = Schema(implementation = OTPVerifyResponse::class) + )] + )] + ) + suspend fun confirmEnableTwoFactor( + @RequestBody request: ConfirmTwoFactorRequest, @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = twoFactorConfigService.confirmEnableTwoFactor( + request.method, + request.otp, + securityContext.authentication.name + ) + return ResponseEntity.ok(response) + } + + @PostMapping("/disable/request") + @Operation( + summary = "Request disabling two-factor authentication", description = """ +POST /v1/2fa/disable/request. + +Security: Bearer token is required. + +Behavior: +Starts the two-factor authentication disable flow for the authenticated user. + +Allowed values: +- method: EMAIL, SMS, TOTP + +Response: +- EMAIL/SMS: Returns the OTP receiver information. An OTP is sent to the selected receiver. +- TOTP: Returns the TOTP code. +""", responses = [ApiResponse( + responseCode = "200", description = "Two-factor enable request created successfully.", content = [Content( + mediaType = "application/json", schema = Schema(implementation = TwoFactorResponse::class) + )] + )] + ) + suspend fun requestDisableTwoFactor( + @RequestBody request: TwoFactorRequest, @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = twoFactorConfigService.requestDisableTwoFactor( + request.method, securityContext.authentication.name + ) + return ResponseEntity.ok(response) + } + + @PostMapping("/disable/confirm") + @Operation( + summary = "Confirm disabling two-factor authentication", description = """ +POST /v1/two-factor/disable/confirm. + +Security: Bearer token is required. + +Behavior: +Confirm the two-factor authentication enable flow for the authenticated user. + +Allowed values: +- method: EMAIL, SMS, TOTP +- otp : String + +Response: +- Returns the otp result. +""", responses = [ApiResponse( + responseCode = "200", description = "Two-factor authentication enabled successfully.", content = [Content( + mediaType = "application/json", schema = Schema(implementation = OTPVerifyResponse::class) + )] + )] + ) + suspend fun confirmDisableTwoFactor( + @RequestBody request: ConfirmTwoFactorRequest, @CurrentSecurityContext securityContext: SecurityContext + ): ResponseEntity { + val response = twoFactorConfigService.confirmDisableTwoFactor( + request.method, + request.otp, + securityContext.authentication.name + ) + return ResponseEntity.ok(response) + + } + + @PostMapping("/totp/setup") + @Operation( + summary = "Setup TOTP (Authenticator App)", + description = """ +POST /v1/user/2fa/totp/setup + +Security: Bearer token is required. + +Behavior: +Generates secret key and setup URL (otpauth://) for setting up Authenticator app (e.g. Google Authenticator). +""", + responses = [ + ApiResponse( + responseCode = "200", + description = "TOTP setup credentials generated successfully.", + content = [ + Content( + mediaType = "application/json", + schema = Schema(implementation = SetupTOTPResponse::class) + ) + ] + ) + ] + ) + suspend fun setupTOTP(@CurrentSecurityContext securityContext: SecurityContext): SetupTOTPResponse { + return twoFactorConfigService.setupTOTP(securityContext.authentication.name) + } + + @PostMapping("/totp/verify") + @Operation( + summary = "Verify TOTP setup code", + description = """ +POST /v1/user/2fa/totp/verify + +Security: Bearer token is required. + +Behavior: +Verifies the generated TOTP code during the initial authenticator setup phase. +""", + responses = [ + ApiResponse( + responseCode = "200", + description = "TOTP setup code verified successfully." + ) + ] + ) + suspend fun verifyTOTPSetup( + @CurrentSecurityContext securityContext: SecurityContext, + @RequestBody request: TOTPCode + ) { + return twoFactorConfigService.verifyTOTPSetup(securityContext.authentication.name, request.code) + } +} \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/OTP.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/OTP.kt index 3cf756c22..feae637b0 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/OTP.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/OTP.kt @@ -28,7 +28,7 @@ data class OTPVerifyResponse( data class TempOtpResponse(val otp: String?, val otpReceiver: OTPReceiver?) enum class OTPAction { - REGISTER, FORGET, NONE + REGISTER, LOGIN, FORGET, NONE, TWO_FACTOR } enum class OTPResultType { diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TOTP.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TOTP.kt new file mode 100644 index 000000000..55c171611 --- /dev/null +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TOTP.kt @@ -0,0 +1,28 @@ +package co.nilin.opex.auth.model + +data class SetupTOTPRequest( + val userId: String, + val label: String? +) + +data class SetupTOTPResponse( + val uri: String +) + +data class VerifyTOTPRequest( + val userId: String, + val code: String +) + +data class VerifyTOTPResponse(val result: Boolean) + +data class TOTPQueryResponse( + val userId: String, + val isEnabled: Boolean, + val isActivated: Boolean, + val uri : String +) + +data class TOTPCode( + val code: String +) diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TwoFactor.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TwoFactor.kt new file mode 100644 index 000000000..bbcf880a7 --- /dev/null +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/TwoFactor.kt @@ -0,0 +1,15 @@ +package co.nilin.opex.auth.model + +import com.fasterxml.jackson.annotation.JsonInclude + +data class TwoFactorRequest( + val method: OTPType, +) + +data class ConfirmTwoFactorRequest( + val method: OTPType, + val otp: String, +) + +@JsonInclude(JsonInclude.Include.NON_NULL) +data class TwoFactorResponse(val otp: String?, val otpReceiver: OTPReceiver?) diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt index 1b5359d0c..1e809a95d 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/model/UserRegister.kt @@ -4,8 +4,6 @@ import co.nilin.opex.auth.data.Device data class RegisterUserRequest( val username: String, - val firstName: String? = null, - val lastName: String? = null, val captchaType: CaptchaType? = CaptchaType.INTERNAL, val captchaCode: String, ) diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/KeycloakProxy.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/KeycloakProxy.kt index 5fe5e21b7..b55d35766 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/KeycloakProxy.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/KeycloakProxy.kt @@ -6,6 +6,7 @@ import co.nilin.opex.auth.model.* import co.nilin.opex.auth.utils.generateRandomID import co.nilin.opex.common.OpexError import co.nilin.opex.common.utils.LoggerDelegate +import jakarta.ws.rs.NotFoundException import kotlinx.coroutines.reactive.awaitFirstOrElse import kotlinx.coroutines.reactive.awaitSingle import kotlinx.coroutines.reactor.awaitSingleOrNull @@ -155,6 +156,28 @@ class KeycloakProxy( return users[0].id } + suspend fun findUserByUuid(uuid: String): KeycloakUser? { + return try { + opexRealm.users() + .get(uuid) + .toRepresentation() + .let { representation -> + KeycloakUser( + id = representation.id, + username = representation.username, + email = representation.email, + firstName = representation.firstName, + lastName = representation.lastName, + emailVerified = representation.isEmailVerified, + enabled = representation.isEnabled, + attributes = representation.attributes + ) + } + } catch (e: Exception) { + null + } + } + suspend fun findUserByUsername(username: Username): KeycloakUser? { val users = findUserByAttribute(username.asAttribute()) return if (users.isEmpty()) null else users[0] @@ -177,8 +200,6 @@ class KeycloakProxy( suspend fun createUser( username: Username, - firstName: String?, - lastName: String?, enabled: Boolean ) { val keycloakUrl = "${keycloakConfig.url}/admin/realms/${keycloakConfig.realm}/users" @@ -192,15 +213,13 @@ class KeycloakProxy( hashMapOf( "username" to internalID, "emailVerified" to enabled, - "firstName" to firstName, - "lastName" to lastName, "enabled" to enabled, "attributes" to hashMapOf( "kycLevel" to "0" ).apply { if (username.type == UsernameType.MOBILE) put("mobile", username.value) - put(Attributes.OTP, OTPType.EMAIL.name + "," + OTPType.SMS.name) + put(Attributes.OTP, OTPType.NONE.name) } ).apply { if (username.type == UsernameType.EMAIL) put("email", username.value) } ) @@ -399,6 +418,20 @@ class KeycloakProxy( } } + suspend fun updateOtpConfig( + userId: String, + otpConfig: String + ) { + updateUserFields( + userId = userId, + updates = mapOf( + "attributes" to mapOf( + Attributes.OTP to otpConfig + ) + ) + ) + } + private suspend fun updateUserFields(userId: String, updates: Map) { val url = "${keycloakConfig.url}/admin/realms/${keycloakConfig.realm}/users/$userId" @@ -411,7 +444,20 @@ class KeycloakProxy( .toMutableMap() updates.forEach { (key, value) -> - existingUser[key] = value + if (key == "attributes" && value is Map<*, *>) { + val currentAttributes = (existingUser["attributes"] as? Map) + ?.toMutableMap() ?: mutableMapOf() + + value.forEach { (attrKey, attrValue) -> + if (attrKey is String && attrValue != null) { + currentAttributes[attrKey] = attrValue + } + } + + existingUser["attributes"] = currentAttributes + } else { + existingUser[key] = value + } } keycloakClient.put() diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/OTPProxy.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/OTPProxy.kt index 3a81f4486..dd8f9ddfa 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/OTPProxy.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/proxy/OTPProxy.kt @@ -1,9 +1,6 @@ package co.nilin.opex.auth.proxy -import co.nilin.opex.auth.model.OTPReceiver -import co.nilin.opex.auth.model.OTPVerifyRequest -import co.nilin.opex.auth.model.OTPVerifyResponse -import co.nilin.opex.auth.model.TempOtpResponse +import co.nilin.opex.auth.model.* import kotlinx.coroutines.reactive.awaitSingle import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.MediaType @@ -11,17 +8,19 @@ import org.springframework.stereotype.Component import org.springframework.web.reactive.function.BodyInserters import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.awaitBody -import org.springframework.web.reactive.function.client.toEntity @Component class OTPProxy(@Qualifier("otpWebClient") private val webClient: WebClient) { - //TODO IMPORTANT: remove in production - - suspend fun requestOTP(userId: String, receivers: List): TempOtpResponse { + suspend fun requestOTP( + userId: String, + receivers: List, + otpAction: OTPAction? = null + ): TempOtpResponse { val request = object { val userId = userId val receivers = receivers + val action = otpAction } return webClient.post().uri("/otp") @@ -47,4 +46,43 @@ class OTPProxy(@Qualifier("otpWebClient") private val webClient: WebClient) { .retrieve() .awaitBody() } + + // ---------------- TOTP ---------------- + + suspend fun setupTOTP(userId: String, label: String): SetupTOTPResponse { + return webClient.post() + .uri("/totp/setup") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(SetupTOTPRequest(userId, label)) + .retrieve() + .awaitBody() + } + + suspend fun verifyTOTPSetup(userId: String, code: String) { + webClient.post() + .uri("/totp/setup/verify") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(VerifyTOTPRequest(userId, code)) + .retrieve() + .toBodilessEntity() + .awaitSingle() + } + + suspend fun verifyTOTP(userId: String, code: String): VerifyTOTPResponse { + return webClient.post() + .uri("/totp/verify") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(VerifyTOTPRequest(userId, code)) + .retrieve() + .awaitBody() + } + + suspend fun queryTOTP(userId: String): TOTPQueryResponse { + return webClient.get() + .uri("/totp/query/$userId") + .retrieve() + .awaitBody() + } + + } \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/ForgetPasswordService.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/ForgetPasswordService.kt index 7856531b3..7aff156c6 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/ForgetPasswordService.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/ForgetPasswordService.kt @@ -32,7 +32,15 @@ class ForgetPasswordService( val otpReceiver = OTPReceiver(uName.value, uName.type.otpType) val user = keycloakProxy.findUserByUsername(uName) ?: return TempOtpResponse("", otpReceiver) //TODO IMPORTANT: remove in production - val result = otpProxy.requestOTP(uName.value, listOf(otpReceiver)) + val result = otpProxy.requestOTP(uName.value, listOf(otpReceiver),OTPAction.FORGET) + return TempOtpResponse(result.otp, otpReceiver) + } + + suspend fun resendForgetOtp(request: ResendOtpRequest): TempOtpResponse { + val uName = Username.create(request.username) + val otpReceiver = OTPReceiver(uName.value, uName.type.otpType) + keycloakProxy.findUserByUsername(uName) ?: return TempOtpResponse("", otpReceiver) + val result = otpProxy.requestOTP(uName.value, listOf(otpReceiver),OTPAction.FORGET) return TempOtpResponse(result.otp, otpReceiver) } @@ -64,4 +72,4 @@ class ForgetPasswordService( } -} +} \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/LoginService.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/LoginService.kt index c41750589..585c516d3 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/LoginService.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/LoginService.kt @@ -34,18 +34,23 @@ class LoginService( request.captchaCode, request.captchaType ?: CaptchaType.INTERNAL ) + val username = Username.create(request.username) - val user = - keycloakProxy.findUserByUsername(username) ?: throw OpexError.UsernameOrPasswordIsIncorrect.exception() - val otpTypes = (user.attributes?.get(Attributes.OTP)?.get(0) ?: OTPType.NONE.name).split(",") + val user = keycloakProxy.findUserByUsername(username) + ?: throw OpexError.UsernameOrPasswordIsIncorrect.exception() + + val otpType = user.attributes?.get(Attributes.OTP)?.firstOrNull() + ?.let { runCatching { OTPType.valueOf(it) }.getOrNull() } + ?: OTPType.NONE - if (otpTypes.contains(OTPType.NONE.name)) { + if (otpType == OTPType.NONE) { val token = keycloakProxy.getUserToken( username, request.password, request.clientId, request.clientSecret ).apply { if (!request.rememberMe) refreshToken = null } + sendLoginEvent(user.id, token.sessionState, request, token.expiresIn) return TokenResponse(token, null, null) } @@ -56,61 +61,114 @@ class LoginService( username, request.password, PRE_AUTH_CLIENT_ID, - preAuthClientSecretKey, + preAuthClientSecretKey ).apply { refreshToken = null refreshExpiresIn = 0 } + return when (otpType) { + OTPType.EMAIL, OTPType.SMS -> { + val destination = when (otpType) { + OTPType.EMAIL -> user.email + OTPType.SMS -> user.mobile + else -> null + } ?: throw OpexError.BadRequest.exception() + + val requiredOtpTypes = listOf(OTPReceiver(destination, otpType)) + val res = otpProxy.requestOTP(destination, requiredOtpTypes, OTPAction.LOGIN) + + TokenResponse( + token = token, + otp = RequiredOTP(otpType, destination), + otpCode = res.otp + ) + } - val usernameType = username.type.otpType - if (!otpTypes.contains((usernameType.name))) throw OpexError.OTPCannotBeRequested.exception() - val requiredOtpTypes = listOf(OTPReceiver(username.value, usernameType)) - val res = otpProxy.requestOTP(username.value, requiredOtpTypes) - val receiver = when (usernameType) { - OTPType.EMAIL -> user.email - OTPType.SMS -> user.mobile - else -> null - } - - + OTPType.TOTP -> { + TokenResponse( + token = token, + otp = RequiredOTP(OTPType.TOTP, user.id), + otpCode = null + ) + } - return TokenResponse(token, RequiredOTP(usernameType, receiver), res.otp) + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() + } } suspend fun resendLoginOtp(request: ResendOtpRequest, uuid: String): ResendOtpResponse { val username = Username.create(request.username) - val usernameType = username.type.otpType - val user = keycloakProxy.findUserByUsername(username) ?: throw OpexError.UserNotFound.exception() + val user = keycloakProxy.findUserByUsername(username) + ?: throw OpexError.UserNotFound.exception() + if (user.id != uuid) throw OpexError.UnAuthorized.exception() - val requiredOtpTypes = listOf(OTPReceiver(username.value, usernameType)) - val res = otpProxy.requestOTP(request.username, requiredOtpTypes) - val receiver = when (usernameType) { - OTPType.EMAIL -> user.email - OTPType.SMS -> user.mobile - else -> null - } - return ResendOtpResponse(RequiredOTP(usernameType, receiver), res.otp) - } + return when (val otpType = user.currentOtpMethod) { + OTPType.EMAIL, OTPType.SMS -> { + val destination = when (otpType) { + OTPType.EMAIL -> user.email + OTPType.SMS -> user.mobile + else -> null + } ?: throw OpexError.BadRequest.exception() + + val requiredOtpTypes = listOf(OTPReceiver(destination, otpType)) + val res = otpProxy.requestOTP(destination, requiredOtpTypes, OTPAction.LOGIN) + + ResendOtpResponse( + otp = RequiredOTP(otpType, destination), + otpCode = res.otp + ) + } + OTPType.TOTP -> { + ResendOtpResponse( + otp = RequiredOTP(OTPType.TOTP, user.id), + otpCode = null + ) + } + + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() + } + } suspend fun confirmGetToken(request: ConfirmPasswordFlowTokenRequest): TokenResponse { val username = Username.create(request.username) - val otpRequest = OTPVerifyRequest(username.value, listOf(OTPCode(request.otp, username.type.otpType))) - val otpResult = otpProxy.verifyOTP(otpRequest) - if (!otpResult.result) { - when (otpResult.type) { - OTPResultType.EXPIRED -> throw OpexError.ExpiredOTP.exception() - else -> throw OpexError.InvalidOTP.exception() + val user = keycloakProxy.findUserByUsername(username) + ?: throw OpexError.UserNotFound.exception() + + when (val otpType = user.currentOtpMethod) { + OTPType.EMAIL, OTPType.SMS -> { + val destination = when (otpType) { + OTPType.EMAIL -> user.email + OTPType.SMS -> user.mobile + else -> null + } ?: throw OpexError.BadRequest.exception() + + val otpRequest = OTPVerifyRequest( + userId = destination, + otpCodes = listOf(OTPCode(request.otp, otpType)) + ) + val otpResult = otpProxy.verifyOTP(otpRequest) + + if (!otpResult.result) { + throw when (otpResult.type) { + OTPResultType.EXPIRED -> OpexError.ExpiredOTP.exception() + else -> OpexError.InvalidOTP.exception() + } + } + } + + OTPType.TOTP -> { + val totpResult = otpProxy.verifyTOTP(userId = user.id, code = request.otp) + if (!totpResult.result) { + throw OpexError.InvalidTOTPCode.exception() + } } + + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() } -// val token = keycloakProxy.exchangeUserToken( -// request.token, request.clientId, -// request.clientSecret, -// request.clientId -// ).apply { if (!request.rememberMe) refreshToken = null } val token = keycloakProxy.getClientBTokenWithBootstrap( bootstrapToken = request.token, clientId = request.clientId, @@ -123,6 +181,13 @@ class LoginService( return TokenResponse(token, null, null) } + // --- Helper Extension --- + private val KeycloakUser.currentOtpMethod: OTPType + get() = attributes?.get(Attributes.OTP) + ?.firstOrNull() + ?.let { runCatching { OTPType.valueOf(it) }.getOrNull() } + ?: OTPType.NONE + suspend fun getToken(tokenRequest: ExternalIdpTokenRequest): TokenResponse { val idToken = tokenRequest.idToken val decodedJWT = googleProxy.validateGoogleToken(idToken) diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt index 5d2799d20..fa3ce1506 100644 --- a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/RegisterService.kt @@ -31,24 +31,24 @@ class RegisterService( request.captchaType ?: CaptchaType.INTERNAL ) val username = Username.create(request.username) - val userStatus = isUserDuplicate(username) + val otpType = username.type.otpType + val otpReceiver = OTPReceiver(request.username, otpType) + val res = otpProxy.requestOTP(request.username, listOf(otpReceiver), OTPAction.REGISTER) + return TempOtpResponse(res.otp, otpReceiver) + } + suspend fun resendRegistrationOtp(request: ResendOtpRequest): TempOtpResponse { + val username = Username.create(request.username) + isUserDuplicate(username) val otpType = username.type.otpType val otpReceiver = OTPReceiver(request.username, otpType) - val res = otpProxy.requestOTP(request.username, listOf(otpReceiver)) -// todo we have to check for duplication usernames after verifying the register otp - if (!userStatus) - keycloakProxy.createUser( - username, - request.firstName, - request.lastName, - false - ) + val res = otpProxy.requestOTP(request.username, listOf(otpReceiver),OTPAction.REGISTER) return TempOtpResponse(res.otp, otpReceiver) } suspend fun verifyRegister(request: VerifyOTPRequest): String { val username = Username.create(request.username) + val userStatus = isUserDuplicate(username) val otpRequest = OTPVerifyRequest(username.value, listOf(OTPCode(request.otp, username.type.otpType))) val otpResult = otpProxy.verifyOTP(otpRequest) if (!otpResult.result) { @@ -57,6 +57,11 @@ class RegisterService( else -> throw OpexError.InvalidOTP.exception() } } + if (!userStatus) + keycloakProxy.createUser( + username, + false + ) return tempTokenService.generateToken(username.value, OTPAction.REGISTER) } diff --git a/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/TwoFactorConfigService.kt b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/TwoFactorConfigService.kt new file mode 100644 index 000000000..8864eaa72 --- /dev/null +++ b/auth-gateway/auth-gateway-app/src/main/kotlin/co/nilin/opex/auth/service/TwoFactorConfigService.kt @@ -0,0 +1,185 @@ +package co.nilin.opex.auth.service + +import co.nilin.opex.auth.model.* +import co.nilin.opex.auth.proxy.KeycloakProxy +import co.nilin.opex.auth.proxy.OTPProxy +import co.nilin.opex.common.OpexError +import co.nilin.opex.common.utils.LoggerDelegate +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service + +@Service +class TwoFactorConfigService( + private val otpProxy: OTPProxy, + private val keycloakProxy: KeycloakProxy, + @Value("\${app.name}") + private val appName: String, +) { + private val logger by LoggerDelegate() + + suspend fun getTwoFactorConfig(uuid: String): OTPType = + getUserByUuid(uuid).currentOtpMethod + + suspend fun requestEnableTwoFactor(method: OTPType, uuid: String): TwoFactorResponse { + validateMethod(method) + val user = getUserByUuid(uuid) + + if (user.currentOtpMethod != OTPType.NONE) { + throw OpexError.InvalidOTPType.exception() + } + + return when (method) { + OTPType.EMAIL, OTPType.SMS -> sendOtpRequest(user, method) + OTPType.TOTP -> { + val totpConfig = otpProxy.queryTOTP(uuid) + if (!totpConfig.isActivated || !totpConfig.isEnabled) { + throw OpexError.TOTPSetupIncomplete.exception() + } + TwoFactorResponse(otp = null, otpReceiver = OTPReceiver("$appName : ${user.username}", OTPType.TOTP)) + } + + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() + } + } + + suspend fun confirmEnableTwoFactor( + method: OTPType, + otpCode: String, + uuid: String + ): OTPVerifyResponse { + val user = getUserByUuid(uuid) + val result = verifyTwoFactorCode(user, method, otpCode) + keycloakProxy.updateOtpConfig(uuid, method.name) + return result + } + + suspend fun requestDisableTwoFactor(method: OTPType, uuid: String): TwoFactorResponse { + validateMethod(method) + val user = getUserByUuid(uuid) + if (user.currentOtpMethod == OTPType.NONE || user.currentOtpMethod != method) { + throw OpexError.InvalidOTPType.exception() + } + return when (method) { + OTPType.EMAIL, OTPType.SMS -> sendOtpRequest(user, method) + OTPType.TOTP -> TwoFactorResponse(otp = null, otpReceiver = OTPReceiver("$appName : ${user.username}", OTPType.TOTP)) + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() + } + } + + suspend fun confirmDisableTwoFactor( + method: OTPType, + otpCode: String, + uuid: String + ): OTPVerifyResponse { + val user = getUserByUuid(uuid) + val result = verifyTwoFactorCode(user, method, otpCode) + + keycloakProxy.updateOtpConfig(uuid, OTPType.NONE.name) + return result + } + + suspend fun setupTOTP(uuid: String): SetupTOTPResponse { + val user = getUserByUuid(uuid) + val totpResponse = otpProxy.queryTOTP(uuid) + return when { + !totpResponse.isEnabled && !totpResponse.isActivated -> otpProxy.setupTOTP( + uuid, + "$appName : ${user.username}" + ) + + totpResponse.isEnabled -> SetupTOTPResponse(totpResponse.uri) + else -> throw OpexError.BadRequest.exception() + } + } + + suspend fun verifyTOTPSetup(uuid: String, code: String) { + val totpConfig = otpProxy.queryTOTP(uuid) + if (totpConfig.isActivated || !totpConfig.isEnabled) { + throw OpexError.TOTPAlreadyRegistered.exception() + } + otpProxy.verifyTOTPSetup(uuid, code) + keycloakProxy.updateOtpConfig(uuid, OTPType.TOTP.name) + } + + // --- Private Helper Methods --- + + private suspend fun verifyTwoFactorCode( + user: KeycloakUser, + method: OTPType, + otpCode: String + ): OTPVerifyResponse { + validateMethod(method) + + return when (method) { + OTPType.EMAIL, OTPType.SMS -> verifyOTP(user, method, otpCode) + OTPType.TOTP -> { + val totpResponse = otpProxy.verifyTOTP(userId = user.id, code = otpCode) + if (!totpResponse.result) throw OpexError.InvalidTOTPCode.exception() + OTPVerifyResponse(result = true, type = OTPResultType.VALID) + } + + OTPType.NONE -> throw OpexError.InvalidOTPType.exception() + } + } + + private suspend fun getUserByUuid(uuid: String): KeycloakUser = + keycloakProxy.findUserByUuid(uuid) ?: throw OpexError.NotFound.exception() + + private fun validateMethod(method: OTPType) { + if (method == OTPType.NONE) throw OpexError.InvalidOTPType.exception() + } + + private suspend fun sendOtpRequest(user: KeycloakUser, method: OTPType): TwoFactorResponse { + val destination = user.getDestinationFor(method) + val receiver = OTPReceiver(destination, method) + + val response = otpProxy.requestOTP( + destination, + listOf(receiver), + OTPAction.TWO_FACTOR + ) + + return TwoFactorResponse( + otp = response.otp, + otpReceiver = receiver + ) + } + + private suspend fun verifyOTP( + user: KeycloakUser, + method: OTPType, + otpCode: String + ): OTPVerifyResponse { + val destination = user.getDestinationFor(method) + + val result = otpProxy.verifyOTP( + OTPVerifyRequest( + userId = destination, + otpCodes = listOf(OTPCode(otpCode, method)) + ) + ) + + if (!result.result) { + throw when (result.type) { + OTPResultType.EXPIRED -> OpexError.ExpiredOTP.exception() + else -> OpexError.InvalidOTP.exception() + } + } + + return result + } + + // --- Extensions --- + + private val KeycloakUser.currentOtpMethod: OTPType + get() = attributes?.get(Attributes.OTP) + ?.firstOrNull() + ?.let { runCatching { OTPType.valueOf(it) }.getOrNull() } + ?: OTPType.NONE + + private fun KeycloakUser.getDestinationFor(method: OTPType): String = when (method) { + OTPType.EMAIL -> email + OTPType.SMS -> mobile + else -> null + } ?: throw OpexError.BadRequest.exception() +} \ No newline at end of file diff --git a/auth-gateway/auth-gateway-app/src/main/resources/application.yml b/auth-gateway/auth-gateway-app/src/main/resources/application.yml index 04ceb4f9e..9ba2e23de 100644 --- a/auth-gateway/auth-gateway-app/src/main/resources/application.yml +++ b/auth-gateway/auth-gateway-app/src/main/resources/application.yml @@ -67,6 +67,7 @@ keycloak: secret: ${ADMIN_CLIENT_SECRET} google-client-id: ${GOOGLE_CLIENT_ID} app: + name: ${APP_NAME:Opex} otp: url: http://opex-otp/v1 captcha: diff --git a/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt b/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt index 27e99a3e5..ef2ad4892 100644 --- a/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt +++ b/common/src/main/kotlin/co/nilin/opex/common/OpexError.kt @@ -152,6 +152,7 @@ enum class OpexError(val code: Int, val message: String?, val status: HttpStatus TOTPSetupIncomplete(12006, "TOTP setup is incomplete", HttpStatus.BAD_REQUEST), TOTPAlreadyRegistered(12007, "User already registered for TOTP", HttpStatus.BAD_REQUEST), OTPDisabled(12008, "OTP for this receiver type is disabled", HttpStatus.INTERNAL_SERVER_ERROR), + InvalidOTPType(12009, "Invalid OTP type", HttpStatus.BAD_REQUEST), //code 12000 profile diff --git a/docker-compose.yml b/docker-compose.yml index 1c7d6e075..c804e40c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -428,6 +428,7 @@ services: - SWAGGER_AUTH_AUTHORITY=${SWAGGER_AUTH_AUTHORITY} - ALLOWED_ORIGINS=${ALLOWED_ORIGINS} - OPEN_API_SERVER_URL=${OPEN_API_SERVER_URL_AUTH} + - APP_NAME=${APP_NAME} volumes: - auth-gateway-keys:/app/keys depends_on: diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/controller/TOTPController.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/controller/TOTPController.kt index 115db992e..7b769fb57 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/controller/TOTPController.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/controller/TOTPController.kt @@ -37,12 +37,7 @@ class TOTPController(private val service: TOTPService) { @GetMapping("/query/{userId}") suspend fun query(@PathVariable userId: String): TOTPQueryResponse { - val totp = service.findTOTP(userId) - return TOTPQueryResponse( - totp?.userId ?: userId, - totp?.isEnabled ?: false, - totp?.isActivated ?: false, - ) + return service.findTOTP(userId) } @DeleteMapping diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/model/TOTPQueryResponse.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/model/TOTPQueryResponse.kt index 81dfa98b5..476f470dc 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/model/TOTPQueryResponse.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/model/TOTPQueryResponse.kt @@ -4,4 +4,5 @@ data class TOTPQueryResponse( val userId: String, val isEnabled: Boolean, val isActivated: Boolean, + val uri : String ) diff --git a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/TOTPService.kt b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/TOTPService.kt index 7ee56a0eb..c85669665 100644 --- a/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/TOTPService.kt +++ b/otp/otp-app/src/main/kotlin/co/nilin/opex/otp/app/service/TOTPService.kt @@ -2,6 +2,7 @@ package co.nilin.opex.otp.app.service import co.nilin.opex.common.OpexError import co.nilin.opex.otp.app.model.TOTP +import co.nilin.opex.otp.app.model.TOTPQueryResponse import co.nilin.opex.otp.app.repository.TOTPConfigRepository import co.nilin.opex.otp.app.repository.TOTPRepository import dev.samstevens.totp.code.DefaultCodeGenerator @@ -55,8 +56,19 @@ class TOTPService( } } - suspend fun findTOTP(userId: String): TOTP? { - return repository.findByUserId(userId) + suspend fun findTOTP(userId: String): TOTPQueryResponse { + val totp = repository.findByUserId(userId) + val config = configRepository.findOne() + val generatedUri = totp?.secret + ?.takeIf { it.isNotBlank() } + ?.let { secret -> generateUri(userId, config.issuer, secret, totp.label) } + ?: "" + return TOTPQueryResponse( + userId = totp?.userId ?: userId, + isEnabled = totp?.isEnabled ?: false, + isActivated = totp?.isActivated ?: false, + uri = generatedUri + ) } private suspend fun generateSecret(): String { From 204eb1b40ce7f98a5d104e53652377057f2cbd51 Mon Sep 17 00:00:00 2001 From: Amir Rajabi <34955519+AmirRajabii@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:25:44 +0330 Subject: [PATCH 21/22] Add chain scanner url (#722) --- .../co/nilin/opex/api/core/inout/ChainInfo.kt | 5 +++-- .../bcgateway/app/controller/AdminController.kt | 5 +++-- .../app/controller/CryptoCurrencyController.kt | 9 ++++----- .../opex/bcgateway/app/dto/AddChainRequest.kt | 3 ++- .../nilin/opex/bcgateway/app/dto/ChainResponse.kt | 5 +++-- .../opex/bcgateway/app/service/AdminService.kt | 2 +- .../co/nilin/opex/bcgateway/core/model/Chain.kt | 4 +++- .../nilin/opex/bcgateway/core/spi/ChainLoader.kt | 7 ++++++- .../ports/postgres/dao/ChainRepository.kt | 8 ++++++-- .../bcgateway/ports/postgres/impl/ChainHandler.kt | 15 ++++++++++----- .../bcgateway/ports/postgres/model/ChainModel.kt | 6 +++++- .../db/migration/V6__add_chain_scanner_urls.sql | 4 ++++ 12 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/resources/db/migration/V6__add_chain_scanner_urls.sql diff --git a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/ChainInfo.kt b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/ChainInfo.kt index e6c0540a6..f1c5805dd 100644 --- a/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/ChainInfo.kt +++ b/api/api-core/src/main/kotlin/co/nilin/opex/api/core/inout/ChainInfo.kt @@ -3,6 +3,7 @@ package co.nilin.opex.api.core.inout data class ChainInfo( val name: String, val addressTypes: String?, - val externalChainScannerUrl: String? = null, - val addressRegex: String? = null + val addressRegex: String? = null, + val transactionScannerUrl: String? = null, + val addressScannerUrl: String? = null ) \ No newline at end of file diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/AdminController.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/AdminController.kt index b774fe834..d538ece5f 100644 --- a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/AdminController.kt +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/AdminController.kt @@ -28,8 +28,9 @@ class AdminController( ChainResponse( c.name, c.addressTypes.map { it.type }.getOrNull(0), - c.externalChinScannerUrl, - c.addressTypes.map { it.addressRegex }.getOrNull(0) + c.addressTypes.map { it.addressRegex }.getOrNull(0), + c.transactionScannerUrl, + c.addressScannerUrl, ) } } diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/CryptoCurrencyController.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/CryptoCurrencyController.kt index 7c2c6d07f..2cd2f2152 100644 --- a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/CryptoCurrencyController.kt +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/controller/CryptoCurrencyController.kt @@ -8,8 +8,6 @@ import co.nilin.opex.bcgateway.core.model.FetchGateways import co.nilin.opex.bcgateway.core.model.WithdrawData import co.nilin.opex.bcgateway.core.spi.ChainLoader import co.nilin.opex.bcgateway.core.spi.CryptoCurrencyHandlerV2 -import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.reactor.ReactorContext import org.springframework.web.bind.annotation.* @RestController @@ -78,9 +76,10 @@ class CryptoCurrencyController( ChainResponse( c.name, c.addressTypes.map { it.type }.getOrNull(0), - c.externalChinScannerUrl, - c.addressTypes.map { it.addressRegex }.getOrNull(0) - ) + c.addressTypes.map { it.addressRegex }.getOrNull(0), + c.transactionScannerUrl, + c.addressScannerUrl, + ) } } diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/AddChainRequest.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/AddChainRequest.kt index ef148cdb3..49e6135d5 100644 --- a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/AddChainRequest.kt +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/AddChainRequest.kt @@ -3,9 +3,10 @@ package co.nilin.opex.bcgateway.app.dto data class AddChainRequest( val name: String?, val addressType: String?, - val scannerEndpoint: String?, val scheduleDelaySeconds: Int, val scheduleErrorDelaySeconds: Int, + val transactionScannerUrl: String? = null, + val addressScannerUrl: String? = null ) { fun isValid(): Boolean { return !name.isNullOrEmpty() && !addressType.isNullOrEmpty() && scheduleDelaySeconds > 0 && scheduleErrorDelaySeconds > 0 diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/ChainResponse.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/ChainResponse.kt index fbb860f7b..e061aa263 100644 --- a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/ChainResponse.kt +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/dto/ChainResponse.kt @@ -3,6 +3,7 @@ package co.nilin.opex.bcgateway.app.dto data class ChainResponse( val name: String, val addressTypes: String?, - val externalChainScannerUrl: String? = null, - val addressRegex: String? = null + val addressRegex: String? = null, + val transactionScannerUrl: String? = null, + val addressScannerUrl: String?= null, ) diff --git a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/service/AdminService.kt b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/service/AdminService.kt index f9ad972c0..756695d4a 100644 --- a/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/service/AdminService.kt +++ b/bc-gateway/bc-gateway-app/src/main/kotlin/co/nilin/opex/bcgateway/app/service/AdminService.kt @@ -30,7 +30,7 @@ class AdminService( @Transactional suspend fun addChain(body: AddChainRequest) { - chainLoader.addChain(body.name!!, body.addressType!!) + chainLoader.addChain(body.name!!, body.addressType!!, body.transactionScannerUrl, body.addressScannerUrl) } suspend fun addAddressType(name: String, addressRegex: String, memoRegex: String?) { diff --git a/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/model/Chain.kt b/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/model/Chain.kt index 1b6c98000..c48ae4522 100644 --- a/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/model/Chain.kt +++ b/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/model/Chain.kt @@ -3,4 +3,6 @@ package co.nilin.opex.bcgateway.core.model data class Chain( val name: String, val addressTypes: List, - val externalChinScannerUrl: String? = null) + val transactionScannerUrl: String? = null, + val addressScannerUrl: String? = null +) diff --git a/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/spi/ChainLoader.kt b/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/spi/ChainLoader.kt index 6c3f6bd4c..57b900372 100644 --- a/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/spi/ChainLoader.kt +++ b/bc-gateway/bc-gateway-core/src/main/kotlin/co/nilin/opex/bcgateway/core/spi/ChainLoader.kt @@ -4,7 +4,12 @@ import co.nilin.opex.bcgateway.core.model.Chain interface ChainLoader { - suspend fun addChain(name: String, addressType: String): Chain + suspend fun addChain( + name: String, + addressType: String, + transactionScannerUrl: String?, + addressScannerUrl: String? + ): Chain suspend fun fetchAllChains(): List diff --git a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/dao/ChainRepository.kt b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/dao/ChainRepository.kt index 32194b3ee..6f7e904ab 100644 --- a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/dao/ChainRepository.kt +++ b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/dao/ChainRepository.kt @@ -11,8 +11,12 @@ import reactor.core.publisher.Mono @Repository interface ChainRepository : ReactiveCrudRepository { - @Query("insert into chains values (:name) on conflict do nothing") - fun insert(name: String): Mono + @Query("insert into chains values (:name,:transaction_scanner_url,:address_scanner_url) on conflict do nothing") + fun insert( + name: String, + transactionScannerUrl: String?, + addressScannerUrl: String? + ): Mono fun findByName(name: String): Mono? diff --git a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/impl/ChainHandler.kt b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/impl/ChainHandler.kt index c14bb6c08..6006172a9 100644 --- a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/impl/ChainHandler.kt +++ b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/impl/ChainHandler.kt @@ -21,7 +21,12 @@ class ChainHandler( private val chainAddressRepository: ChainAddressTypeRepository ) : ChainLoader { - override suspend fun addChain(name: String, addressType: String): Chain { + override suspend fun addChain( + name: String, + addressType: String, + transactionScannerUrl: String?, + addressScannerUrl: String? + ): Chain { val chain = chainRepository.findByName(name)?.awaitFirstOrNull() if (chain != null) throw OpexError.BadRequest.exception() @@ -29,10 +34,10 @@ class ChainHandler( val type = addressTypeRepository.findByType(addressType).awaitFirstOrNull() ?: throw OpexError.InvalidAddressType.exception() - chainRepository.insert(name).awaitFirstOrNull() + chainRepository.insert(name, transactionScannerUrl, addressScannerUrl).awaitFirstOrNull() val model = chainRepository.findByName(name)?.awaitFirstOrElse { throw OpexError.BadRequest.exception() } chainAddressRepository.save(ChainAddressTypeModel(null, model!!.name, type.id!!)).awaitFirstOrNull() - return Chain(model.name, emptyList()) + return Chain(model.name, emptyList(), transactionScannerUrl, addressScannerUrl) } override suspend fun fetchAllChains(): List { @@ -44,7 +49,7 @@ class ChainHandler( .map { AddressType(it.id!!, it.type, it.addressRegex, it.memoRegex) } .toList() - Chain(c.name, addressTypes, c.externalChainScannerUrl) + Chain(c.name, addressTypes, c.transactionScannerUrl, c.addressScannerUrl) } } @@ -52,7 +57,7 @@ class ChainHandler( val chainDao = chainRepository.findByName(chain)?.awaitFirstOrElse { throw OpexError.ChainNotFound.exception() } val addressTypes = chainRepository.findAddressTypesByName(chain) .map { AddressType(it.id!!, it.type, it.addressRegex, it.memoRegex) }.toList() - return Chain(chainDao!!.name, addressTypes) + return Chain(chainDao!!.name, addressTypes, chainDao.transactionScannerUrl, chainDao.addressScannerUrl) } } diff --git a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/model/ChainModel.kt b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/model/ChainModel.kt index 9c3d8c4b8..77c4806b1 100644 --- a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/model/ChainModel.kt +++ b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/kotlin/co/nilin/opex/bcgateway/ports/postgres/model/ChainModel.kt @@ -4,4 +4,8 @@ import org.springframework.data.annotation.Id import org.springframework.data.relational.core.mapping.Table @Table("chains") -data class ChainModel(@Id val name: String, val externalChainScannerUrl: String?) +data class ChainModel( + @Id val name: String, + val transactionScannerUrl: String?, + val addressScannerUrl: String? +) diff --git a/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/resources/db/migration/V6__add_chain_scanner_urls.sql b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/resources/db/migration/V6__add_chain_scanner_urls.sql new file mode 100644 index 000000000..1c08e26ad --- /dev/null +++ b/bc-gateway/bc-gateway-ports/bc-gateway-persister-postgres/src/main/resources/db/migration/V6__add_chain_scanner_urls.sql @@ -0,0 +1,4 @@ +Alter table chains drop column external_chain_scanner_url; + +Alter table chains add column transaction_scanner_url VARCHAR(150); +Alter table chains add column address_scanner_url VARCHAR(150); From ab829c00704c20fe4baa74e2ab9a88c728a31b84 Mon Sep 17 00:00:00 2001 From: Fatemeh imani <46007372+fatemeh-i@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:47:07 +0200 Subject: [PATCH 22/22] Chore/enhance market overview data --- .../ports/proxy/impl/MarketDataProxyImpl.kt | 2 +- .../ports/postgres/dao/TradeRepository.kt | 69 ++++++------- .../postgres/impl/MarketQueryHandlerImpl.kt | 62 +++++++++--- .../src/main/resources/schema.sql | 1 + .../postgres/impl/MarketQueryHandlerTest.kt | 96 ++++++++++++++++++- 5 files changed, 174 insertions(+), 56 deletions(-) diff --git a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketDataProxyImpl.kt b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketDataProxyImpl.kt index 41782c441..3791ef458 100644 --- a/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketDataProxyImpl.kt +++ b/api/api-ports/api-proxy-rest/src/main/kotlin/co/nilin/opex/api/ports/proxy/impl/MarketDataProxyImpl.kt @@ -56,7 +56,7 @@ class MarketDataProxyImpl(@Qualifier("generalWebClient") private val webClient: .onStatus({ t -> t.isError }, { it.createException() }) .bodyToMono() .awaitSingleOrNull() - ?: PriceChange(symbol, openTime = Date().time, closeTime = interval.getTime()) + ?: PriceChange(symbol, openTime = interval.getTime(), closeTime = Date().time) } } diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt index 517db2d74..8c6cd0a5a 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/dao/TradeRepository.kt @@ -189,26 +189,21 @@ interface TradeRepository : ReactiveCrudRepository { select symbol, (select matched_price from last_trade where symbol=t.symbol) - (select matched_price from first_trade where symbol=t.symbol) as price_change, ((((select matched_price from last_trade where symbol=t.symbol) - (select matched_price from first_trade where symbol=t.symbol))/(select matched_price from first_trade where symbol=t.symbol))*100) as price_change_percent, - (sum(matched_quantity)/sum(matched_price)) as weighted_avg_price, + (sum(matched_price * matched_quantity)/nullif(sum(matched_quantity), 0)) as weighted_avg_price, (select matched_price from last_trade where symbol=t.symbol) as last_price, (select matched_quantity from last_trade where symbol=t.symbol) as last_qty, ( - select price from orders + select max(price) from orders inner join open_orders oo on orders.ouid = oo.ouid where create_date > :date and symbol=t.symbol and side='BID' - order by create_date desc limit 1 ) as bid_price, ( - select price from orders + select min(price) from orders inner join open_orders oo on orders.ouid = oo.ouid where create_date > :date and symbol=t.symbol and side='ASK' - order by create_date desc limit 1 ) as ask_price, ( - select price from orders - inner join open_orders oo on orders.ouid = oo.ouid - where create_date > :date and symbol=t.symbol - order by create_date desc limit 1 + select matched_price from first_trade where symbol=t.symbol ) as open_price, max(matched_price) as high_price, min(matched_price) as low_price, @@ -230,26 +225,21 @@ interface TradeRepository : ReactiveCrudRepository { select symbol, (select matched_price from last_trade) - (select matched_price from first_trade) as price_change, ((((select matched_price from last_trade) - (select matched_price from first_trade))/(select matched_price from first_trade))*100) as price_change_percent, - (sum(matched_quantity)/sum(matched_price)) as weighted_avg_price, + (sum(matched_price * matched_quantity)/nullif(sum(matched_quantity), 0)) as weighted_avg_price, (select matched_price from last_trade) as last_price, (select matched_quantity from last_trade) as last_qty, ( - select price from orders + select max(price) from orders inner join open_orders oo on orders.ouid = oo.ouid where create_date > :date and symbol=t.symbol and side='BID' - order by create_date desc limit 1 ) as bid_price, ( - select price from orders + select min(price) from orders inner join open_orders oo on orders.ouid = oo.ouid where create_date > :date and symbol=t.symbol and side='ASK' - order by create_date desc limit 1 ) as ask_price, ( - select price from orders - inner join open_orders oo on orders.ouid = oo.ouid - where create_date > :date and symbol=t.symbol - order by create_date desc limit 1 + select matched_price from first_trade ) as open_price, max(matched_price) as high_price, min(matched_price) as low_price, @@ -350,29 +340,35 @@ interface TradeRepository : ReactiveCrudRepository { :interval::INTERVAL ) ), + limited_intervals AS ( + SELECT * + FROM intervals + ORDER BY start_time DESC + LIMIT :limit + ), first_trade AS ( - SELECT DISTINCT ON (f.start_time) - f.start_time, - f.end_time, + SELECT DISTINCT ON (i.start_time) + i.start_time, + i.end_time, t.matched_price AS open_price - FROM intervals f + FROM limited_intervals i LEFT JOIN trades t - ON t.create_date >= f.start_time - AND t.create_date < f.end_time + ON t.create_date >= i.start_time + AND t.create_date < i.end_time AND t.symbol = :symbol - ORDER BY f.start_time, t.create_date + ORDER BY i.start_time, t.create_date ), last_trade AS ( - SELECT DISTINCT ON (f.start_time) - f.start_time, - f.end_time, + SELECT DISTINCT ON (i.start_time) + i.start_time, + i.end_time, t.matched_price AS close_price - FROM intervals f + FROM limited_intervals i LEFT JOIN trades t - ON t.create_date >= f.start_time - AND t.create_date < f.end_time + ON t.create_date >= i.start_time + AND t.create_date < i.end_time AND t.symbol = :symbol - ORDER BY f.start_time, t.create_date DESC + ORDER BY i.start_time, t.create_date DESC ), ohlcv AS ( SELECT @@ -384,7 +380,7 @@ interface TradeRepository : ReactiveCrudRepository { lt.close_price AS close, SUM(t.matched_quantity) AS volume, COUNT(t.id) AS trades - FROM intervals i + FROM limited_intervals i LEFT JOIN trades t ON t.create_date >= i.start_time AND t.create_date < i.end_time @@ -396,12 +392,7 @@ interface TradeRepository : ReactiveCrudRepository { GROUP BY i.start_time, i.end_time, ft.open_price, lt.close_price ) SELECT * - FROM ( - SELECT * - FROM ohlcv - ORDER BY open_time DESC - limit :limit - ) sub + FROM ohlcv ORDER BY open_time ASC """ ) diff --git a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerImpl.kt b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerImpl.kt index fe8673002..5bce4f092 100644 --- a/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerImpl.kt +++ b/market/market-ports/market-persister-postgres/src/main/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerImpl.kt @@ -21,6 +21,7 @@ import java.math.BigDecimal import java.time.Instant import java.time.LocalDateTime import java.time.ZoneId +import java.time.temporal.ChronoUnit import java.util.* @@ -35,19 +36,23 @@ class MarketQueryHandlerImpl( override suspend fun getTradeTickerData(interval: Interval): List { return redisCacheHelper.getOrElse("tradeTickerData:${interval.label}", 2.minutes()) { + val closeTime = Date().time + val openTime = interval.getTime() tradeRepository.tradeTicker(interval.getLocalDateTime()) .collectList() .awaitFirstOrElse { emptyList() } - .map { it.asPriceChangeResponse(Date().time, interval.getTime()) } + .map { it.asPriceChangeResponse(openTime, closeTime) } } } override suspend fun getTradeTickerDateBySymbol(symbol: String, interval: Interval): PriceChange? { val cacheId = "tradeTickerData:$symbol:${interval.label}" return redisCacheHelper.getOrElse(cacheId, 2.minutes()) { + val closeTime = Date().time + val openTime = interval.getTime() tradeRepository.tradeTickerBySymbol(symbol, interval.getLocalDateTime()) .awaitSingleOrNull() - ?.asPriceChangeResponse(Date().time, interval.getTime()) + ?.asPriceChangeResponse(openTime, closeTime) } } @@ -283,21 +288,22 @@ class MarketQueryHandlerImpl( endTime: Long?, limit: Int, ): List { - val st = if (startTime == null) - tradeRepository.findFirstByCreateDate().awaitSingleOrNull()?.createDate ?: LocalDateTime.now() + val intervalStep = parseIntervalStep(interval) + val latestTradeDate = if (startTime == null || endTime == null) + tradeRepository.findLastByCreateDate().awaitSingleOrNull()?.createDate else - with(Instant.ofEpochMilli(startTime)) { - LocalDateTime.ofInstant(this, ZoneId.systemDefault()) - } - - val et = if (endTime == null) - tradeRepository.findLastByCreateDate().awaitSingleOrNull()?.createDate ?: LocalDateTime.now() - else - with(Instant.ofEpochMilli(endTime)) { - LocalDateTime.ofInstant(this, ZoneId.systemDefault()) - } + null + val fallbackDate = latestTradeDate ?: LocalDateTime.now() + val startDate = startTime?.asLocalDateTime() ?: when { + endTime != null -> shiftByIntervals(endTime.asLocalDateTime(), intervalStep, -(limit - 1).toLong()) + else -> shiftByIntervals(fallbackDate, intervalStep, -(limit - 1).toLong()) + } + val endDate = endTime?.asLocalDateTime() ?: when { + startTime != null -> shiftByIntervals(startDate, intervalStep, (limit - 1).toLong()) + else -> fallbackDate + } - return tradeRepository.candleData(symbol, interval, st, et, limit) + return tradeRepository.candleData(symbol, interval, startDate, endDate, limit) .collectList() .awaitFirstOrElse { emptyList() } .map { @@ -457,6 +463,32 @@ class MarketQueryHandlerImpl( count ?: 0 ) + private fun Long.asLocalDateTime(): LocalDateTime = with(Instant.ofEpochMilli(this)) { + LocalDateTime.ofInstant(this, ZoneId.systemDefault()) + } + + private fun parseIntervalStep(interval: String): Pair { + val parts = interval.trim().split(Regex("\\s+"), limit = 2) + val amount = parts.firstOrNull()?.toLongOrNull() + ?: throw IllegalArgumentException("Invalid interval amount: $interval") + val unit = when (parts.getOrNull(1)?.uppercase(Locale.US)?.removeSuffix("S")) { + "MINUTE" -> ChronoUnit.MINUTES + "HOUR" -> ChronoUnit.HOURS + "DAY" -> ChronoUnit.DAYS + else -> throw IllegalArgumentException("Unsupported interval unit: $interval") + } + return amount to unit + } + + private fun shiftByIntervals( + dateTime: LocalDateTime, + intervalStep: Pair, + intervals: Long, + ): LocalDateTime { + val (amount, unit) = intervalStep + return dateTime.plus(intervals * amount, unit) + } + private fun Long.approximate(): Long { if (this < 10) return this diff --git a/market/market-ports/market-persister-postgres/src/main/resources/schema.sql b/market/market-ports/market-persister-postgres/src/main/resources/schema.sql index 294115909..2395840f1 100644 --- a/market/market-ports/market-persister-postgres/src/main/resources/schema.sql +++ b/market/market-ports/market-persister-postgres/src/main/resources/schema.sql @@ -71,6 +71,7 @@ CREATE TABLE IF NOT EXISTS trades ); CREATE INDEX IF NOT EXISTS idx_trades_symbol on trades (symbol); CREATE INDEX IF NOT EXISTS idx_trades_create_date on trades (create_date); +CREATE INDEX IF NOT EXISTS idx_trades_symbol_create_date on trades (symbol, create_date); ALTER TABLE trades ALTER COLUMN id TYPE BIGINT, diff --git a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerTest.kt b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerTest.kt index bb25377ae..b0d479467 100644 --- a/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerTest.kt +++ b/market/market-ports/market-persister-postgres/src/test/kotlin/co/nilin/opex/market/ports/postgres/impl/MarketQueryHandlerTest.kt @@ -1,5 +1,6 @@ package co.nilin.opex.market.ports.postgres.impl +import co.nilin.opex.common.utils.Interval import co.nilin.opex.market.core.inout.MarketTrade import co.nilin.opex.market.core.inout.Order import co.nilin.opex.market.core.inout.OrderDirection @@ -8,7 +9,10 @@ import co.nilin.opex.market.ports.postgres.dao.OrderRepository import co.nilin.opex.market.ports.postgres.dao.OrderStatusRepository import co.nilin.opex.market.ports.postgres.dao.TradeRepository import co.nilin.opex.market.ports.postgres.impl.sample.VALID +import co.nilin.opex.market.ports.postgres.model.CandleInfoData import co.nilin.opex.market.ports.postgres.model.LastPrice +import co.nilin.opex.market.ports.postgres.model.TradeModel +import co.nilin.opex.market.ports.postgres.model.TradeTickerData import co.nilin.opex.market.ports.postgres.util.RedisCacheHelper import io.mockk.coEvery import io.mockk.every @@ -18,6 +22,8 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import reactor.core.publisher.Flux import reactor.core.publisher.Mono +import java.math.BigDecimal +import java.time.LocalDateTime class MarketQueryHandlerTest { private val orderRepository = mockk() @@ -132,5 +138,93 @@ class MarketQueryHandlerTest { assertThat(marketTradeResponses?.count()).isEqualTo(1) assertThat(marketTradeResponses?.first()).isEqualTo(VALID.MARKET_TRADE_RESPONSE) } -} + @Test + fun givenTickerData_whenTradeTickerRequested_thenTickerTimeWindowIsOrderedCorrectly(): Unit = runBlocking { + val tradeTickerData = TradeTickerData( + VALID.ETH_USDT, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.ONE, + BigDecimal.TEN, + BigDecimal.ONE, + BigDecimal.TEN, + 1L, + 2L, + 3L + ) + coEvery { + redisCacheHelper.getOrElse>( + eq("tradeTickerData:${Interval.TwentyFourHours.label}"), + any(), + any() + ) + } coAnswers { + thirdArg List>().invoke() + } + every { tradeRepository.tradeTicker(any()) } returns Flux.just(tradeTickerData) + + val priceChanges = marketQueryHandler.getTradeTickerData(Interval.TwentyFourHours) + + assertThat(priceChanges).hasSize(1) + assertThat(priceChanges.first().openTime).isLessThanOrEqualTo(priceChanges.first().closeTime) + } + + @Test + fun givenMissingCandleBounds_whenGetCandleInfo_thenOnlyLatestIntervalsAreRequested(): Unit = runBlocking { + val latestTradeDate = LocalDateTime.of(2024, 1, 1, 10, 15) + val expectedStartDate = latestTradeDate.minusHours(2) + val latestTrade = TradeModel( + 1L, + 1L, + VALID.ETH_USDT, + "ETH", + "USDT", + BigDecimal.TEN, + BigDecimal.ONE, + BigDecimal.TEN, + BigDecimal.TEN, + BigDecimal.ZERO, + BigDecimal.ZERO, + "ETH", + "USDT", + latestTradeDate, + "maker", + "taker", + "maker-user", + "taker-user", + latestTradeDate + ) + val candleInfo = CandleInfoData( + expectedStartDate, + expectedStartDate.plusHours(1), + BigDecimal.ONE, + BigDecimal.TWO, + BigDecimal.TWO, + BigDecimal.ONE, + BigDecimal.TEN, + 1 + ) + coEvery { tradeRepository.findLastByCreateDate() } returns Mono.just(latestTrade) + coEvery { + tradeRepository.candleData( + VALID.ETH_USDT, + "1 HOURS", + expectedStartDate, + latestTradeDate, + 3 + ) + } returns Flux.just(candleInfo) + + val candles = marketQueryHandler.getCandleInfo(VALID.ETH_USDT, "1 HOURS", null, null, 3) + + assertThat(candles).hasSize(1) + assertThat(candles.first().openTime).isEqualTo(expectedStartDate) + assertThat(candles.first().closeTime).isEqualTo(expectedStartDate.plusHours(1)) + } +}