From 99d13178ce40f579f6b41c25a3bbd823eb0cb83c Mon Sep 17 00:00:00 2001 From: Ahmad Tash <144666528+AhmadTash@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:18:33 +0300 Subject: [PATCH 1/6] fix(connections): reject unresolved OAuth2 placeholders before they reach the provider (#14832) --- packages/core/shared/package.json | 2 +- .../app-connection/app-connection.ts | 3 +- .../common/resolve-value-from-props.test.ts | 23 +++ .../oauth2/oauth2-util.ts | 44 ++++- .../app-connection/oauth2-placeholder.test.ts | 160 ++++++++++++++++++ .../oauth2-connection-settings.tsx | 19 ++- 6 files changed, 241 insertions(+), 10 deletions(-) create mode 100644 packages/core/shared/test/common/resolve-value-from-props.test.ts create mode 100644 packages/server/api/test/integration/ce/app-connection/oauth2-placeholder.test.ts diff --git a/packages/core/shared/package.json b/packages/core/shared/package.json index 8a0cb8f37d1d..131bfef10444 100644 --- a/packages/core/shared/package.json +++ b/packages/core/shared/package.json @@ -1,6 +1,6 @@ { "name": "@activepieces/shared", - "version": "0.138.0", + "version": "0.138.1", "type": "commonjs", "sideEffects": false, "main": "./dist/src/index.js", diff --git a/packages/core/shared/src/lib/automation/app-connection/app-connection.ts b/packages/core/shared/src/lib/automation/app-connection/app-connection.ts index 462a48938862..fd7dcada4f50 100755 --- a/packages/core/shared/src/lib/automation/app-connection/app-connection.ts +++ b/packages/core/shared/src/lib/automation/app-connection/app-connection.ts @@ -149,14 +149,13 @@ export const AppConnectionOwners = z.object({ }) export type AppConnectionOwners = z.infer -/**i.e props: {projectId: "123"} and value: "{{projectId}}" will return "123" */ export const resolveValueFromProps = (props: Record | undefined, value: string)=>{ let resolvedScope = value if (!props) { return resolvedScope } Object.entries(props).forEach(([key, value]) => { - resolvedScope = resolvedScope.replace(`{${key}}`, String(value)) + resolvedScope = resolvedScope.replaceAll(`{${key}}`, () => String(value)) }) return resolvedScope } diff --git a/packages/core/shared/test/common/resolve-value-from-props.test.ts b/packages/core/shared/test/common/resolve-value-from-props.test.ts new file mode 100644 index 000000000000..83f3aea19511 --- /dev/null +++ b/packages/core/shared/test/common/resolve-value-from-props.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { resolveValueFromProps } from '../../src/lib/automation/app-connection/app-connection' + +describe('resolveValueFromProps', () => { + it('substitutes every occurrence of a placeholder, not just the first', () => { + expect(resolveValueFromProps({ tenant: 'contoso' }, 'https://host/{tenant}/x/{tenant}')) + .toBe('https://host/contoso/x/contoso') + }) + + it('inserts prop values literally instead of expanding replacement patterns', () => { + expect(resolveValueFromProps({ tenant: 'a$&b' }, 'https://host/{tenant}')) + .toBe('https://host/a$&b') + }) + + it('leaves placeholders that have no matching prop untouched', () => { + expect(resolveValueFromProps({ cloud: 'login.microsoftonline.com' }, 'https://{cloud}/{tenant}/token')) + .toBe('https://login.microsoftonline.com/{tenant}/token') + }) + + it('returns the value unchanged when there are no props', () => { + expect(resolveValueFromProps(undefined, 'https://{cloud}/token')).toBe('https://{cloud}/token') + }) +}) diff --git a/packages/server/api/src/app/app-connection/app-connection-service/oauth2/oauth2-util.ts b/packages/server/api/src/app/app-connection/app-connection-service/oauth2/oauth2-util.ts index 4804f71d35d3..d9ceb18ffae3 100644 --- a/packages/server/api/src/app/app-connection/app-connection-service/oauth2/oauth2-util.ts +++ b/packages/server/api/src/app/app-connection/app-connection-service/oauth2/oauth2-util.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'crypto' -import { ActivepiecesError, assertNotNullOrUndefined, deleteProps, ErrorCode, PlatformId } from '@activepieces/core-utils' -import { PropertyType } from '@activepieces/pieces-framework' +import { ActivepiecesError, assertNotNullOrUndefined, deleteProps, ErrorCode, isNil, PlatformId, unique } from '@activepieces/core-utils' +import { OAuth2Props, PropertyType } from '@activepieces/pieces-framework' import { AppConnection, AppConnectionType, BaseOAuth2ConnectionValue, GetOAuth2AuthorizationUrlResponse, OAuth2GrantType, resolveValueFromProps } from '@activepieces/shared' import { isAxiosError } from 'axios' import { FastifyBaseLogger } from 'fastify' @@ -78,6 +78,11 @@ export const oauth2Util = (log: FastifyBaseLogger) => ({ assertNotNullOrUndefined(pieceAuth, 'auth') switch (pieceAuth.type) { case PropertyType.OAUTH2: + assertPlaceholdersResolved({ + templates: [pieceAuth.tokenUrl, ...pieceAuth.scope], + props, + authProps: pieceAuth.props, + }) return resolveValueFromProps(props, pieceAuth.tokenUrl) default: throw new ActivepiecesError({ @@ -120,8 +125,13 @@ export const oauth2Util = (log: FastifyBaseLogger) => ({ throwOnFailure: true, projectIds: projectId ? [projectId] : undefined, }) - const authUrl = resolveValueFromProps(props, pieceAuth.authUrl) const selectedScopes = resolveSelectedScopes(scopes, pieceAuth.scope) + assertPlaceholdersResolved({ + templates: [pieceAuth.authUrl, ...selectedScopes], + props, + authProps: pieceAuth.props, + }) + const authUrl = resolveValueFromProps(props, pieceAuth.authUrl) const scope = resolveValueFromProps(props, selectedScopes.join(' ')) const queryParams: Record = { @@ -213,6 +223,34 @@ const resolveSelectedScopes = (requested: string[] | undefined, allowed: string[ return requested } +const assertPlaceholdersResolved = ({ templates, props, authProps }: AssertPlaceholdersResolvedParams): void => { + const declaredProps = authProps ?? {} + const missing = unique( + templates + .flatMap(template => [...template.matchAll(/\{([A-Za-z0-9_]+)\}/g)]) + .map(match => match[1]) + .filter(key => !isNil(declaredProps[key])) + .filter(key => { + const value = props?.[key] + return isNil(value) || String(value).trim() === '' + }), + ) + if (missing.length === 0) { + return + } + const labels = missing.map(key => declaredProps[key].displayName).join(', ') + throw new ActivepiecesError({ + code: ErrorCode.INVALID_APP_CONNECTION, + params: { error: `missing required connection settings: ${labels}` }, + }) +} + +type AssertPlaceholdersResolvedParams = { + templates: string[] + props: Record | undefined + authProps: OAuth2Props | undefined +} + type BuildAuthorizationUrlParams = { platformId: PlatformId pieceName: string diff --git a/packages/server/api/test/integration/ce/app-connection/oauth2-placeholder.test.ts b/packages/server/api/test/integration/ce/app-connection/oauth2-placeholder.test.ts new file mode 100644 index 000000000000..517e40b21cde --- /dev/null +++ b/packages/server/api/test/integration/ce/app-connection/oauth2-placeholder.test.ts @@ -0,0 +1,160 @@ +import { apId, ErrorCode } from '@activepieces/core-utils' +import { PropertyType } from '@activepieces/pieces-framework' +import { PackageType, PieceType } from '@activepieces/shared' +import { FastifyBaseLogger, FastifyInstance } from 'fastify' +import { oauth2Util } from '../../../../src/app/app-connection/app-connection-service/oauth2/oauth2-util' +import { db } from '../../../helpers/db' +import { createMockPieceMetadata } from '../../../helpers/mocks' +import { setupTestEnvironment, teardownTestEnvironment } from '../../../helpers/test-setup' + +let app: FastifyInstance | null = null +let mockLog: FastifyBaseLogger + +beforeAll(async () => { + app = await setupTestEnvironment() + mockLog = app!.log! +}) + +afterAll(async () => { + await teardownTestEnvironment() +}) + +const shortText = (displayName: string) => ({ + type: PropertyType.SHORT_TEXT, + displayName, + required: true, +}) + +const saveOAuth2Piece = async ({ platformId, tokenUrl, scope, props }: { + platformId: string + tokenUrl: string + scope: string[] + props: Record +}): Promise => { + const pieceName = `piece-${apId()}` + await db.save('piece_metadata', createMockPieceMetadata({ + name: pieceName, + version: '1.0.0', + platformId, + pieceType: PieceType.CUSTOM, + packageType: PackageType.REGISTRY, + minimumSupportedRelease: '0.0.0', + maximumSupportedRelease: '999.999.999', + auth: { + type: PropertyType.OAUTH2, + displayName: 'Connection', + required: true, + authUrl: 'https://{cloud}/{tenant}/oauth2/v2.0/authorize', + tokenUrl, + scope, + props, + }, + })) + return pieceName +} + +describe('OAuth2 unresolved placeholder guard', () => { + it('rejects a token url whose placeholder has no matching prop, naming the prop label', async () => { + const platformId = apId() + const pieceName = await saveOAuth2Piece({ + platformId, + tokenUrl: 'https://{cloud}/{tenant}/oauth2/v2.0/token', + scope: ['Mail.Read'], + props: { cloud: shortText('Cloud Environment'), tenant: shortText('Tenant ID') }, + }) + + await expect(oauth2Util(mockLog).getOAuth2TokenUrl({ + platformId, + pieceName, + pieceVersion: '1.0.0', + props: { cloud: 'login.microsoftonline.com' }, + })).rejects.toMatchObject({ + error: { + code: ErrorCode.INVALID_APP_CONNECTION, + params: { error: expect.stringContaining('Tenant ID') }, + }, + }) + }) + + it('rejects a prop that is present but empty', async () => { + const platformId = apId() + const pieceName = await saveOAuth2Piece({ + platformId, + tokenUrl: 'https://{cloud}/{tenant}/oauth2/v2.0/token', + scope: ['Mail.Read'], + props: { cloud: shortText('Cloud Environment'), tenant: shortText('Tenant ID') }, + }) + + await expect(oauth2Util(mockLog).getOAuth2TokenUrl({ + platformId, + pieceName, + pieceVersion: '1.0.0', + props: { cloud: 'login.microsoftonline.com', tenant: ' ' }, + })).rejects.toMatchObject({ + error: { + code: ErrorCode.INVALID_APP_CONNECTION, + params: { error: expect.stringContaining('Tenant ID') }, + }, + }) + }) + + it('rejects a placeholder that only appears in the declared scope', async () => { + const platformId = apId() + const pieceName = await saveOAuth2Piece({ + platformId, + tokenUrl: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', + scope: ['{accessMode}'], + props: { accessMode: shortText('Access Mode') }, + }) + + await expect(oauth2Util(mockLog).getOAuth2TokenUrl({ + platformId, + pieceName, + pieceVersion: '1.0.0', + props: {}, + })).rejects.toMatchObject({ + error: { + code: ErrorCode.INVALID_APP_CONNECTION, + params: { error: expect.stringContaining('Access Mode') }, + }, + }) + }) + + it('resolves the token url when every placeholder is supplied', async () => { + const platformId = apId() + const pieceName = await saveOAuth2Piece({ + platformId, + tokenUrl: 'https://{cloud}/{tenant}/oauth2/v2.0/token', + scope: ['Mail.Read'], + props: { cloud: shortText('Cloud Environment'), tenant: shortText('Tenant ID') }, + }) + + const tokenUrl = await oauth2Util(mockLog).getOAuth2TokenUrl({ + platformId, + pieceName, + pieceVersion: '1.0.0', + props: { cloud: 'login.microsoftonline.com', tenant: 'common' }, + }) + + expect(tokenUrl).toBe('https://login.microsoftonline.com/common/oauth2/v2.0/token') + }) + + it('accepts braces that come from a prop value rather than the template', async () => { + const platformId = apId() + const pieceName = await saveOAuth2Piece({ + platformId, + tokenUrl: '{tokenUrl}', + scope: ['{scopes}'], + props: { tokenUrl: shortText('Token URL'), scopes: shortText('Scopes') }, + }) + + const tokenUrl = await oauth2Util(mockLog).getOAuth2TokenUrl({ + platformId, + pieceName, + pieceVersion: '1.0.0', + props: { tokenUrl: 'https://id.example.com/{realm}/token', scopes: 'openid' }, + }) + + expect(tokenUrl).toBe('https://id.example.com/{realm}/token') + }) +}) diff --git a/packages/web/src/app/connections/oauth2-connection-settings.tsx b/packages/web/src/app/connections/oauth2-connection-settings.tsx index 5ff557cdd785..c63e013fbd27 100644 --- a/packages/web/src/app/connections/oauth2-connection-settings.tsx +++ b/packages/web/src/app/connections/oauth2-connection-settings.tsx @@ -6,8 +6,10 @@ import { PieceMetadataModelSummary, } from '@activepieces/pieces-framework'; import { + ApErrorParams, ApFlagId, AppConnectionType, + ErrorCode, OAuth2GrantType, UpsertCloudOAuth2Request, UpsertOAuth2Request, @@ -39,6 +41,7 @@ import { Input } from '@/components/ui/input'; import { OAuth2App, oauth2Utils } from '@/features/connections'; import { appConnectionsApi } from '@/features/connections/api/app-connections'; import { flagsHooks } from '@/hooks/flags-hooks'; +import { api } from '@/lib/api'; import { cn } from '@/lib/utils'; import { GenericPropertiesForm } from '../builder/piece-properties/generic-properties-form'; @@ -329,12 +332,20 @@ async function openPopup({ authorizationUrl = result.authorizationUrl; codeVerifier = result.codeVerifier; } catch (error: unknown) { - form.setError('request.value.client_id', { + const apError = api.isError(error) + ? (error.response?.data as ApErrorParams | undefined) + : undefined; + form.setError('request.value.code', { type: 'manual', message: - error instanceof Error - ? error.message - : 'Failed to initiate OAuth2 authentication', + apError?.code === ErrorCode.INVALID_APP_CONNECTION + ? t('Connection failed with error {msg}', { + msg: apError.params.error, + }) + : api.extractServerErrorMessage( + error, + 'Failed to initiate OAuth2 authentication', + ), }); setLoading(false); return; From 7694873f9897da255c859b79542ac85c20ec4e63 Mon Sep 17 00:00:00 2001 From: Alexandru Andronic Date: Tue, 18 Aug 2026 16:39:50 +0300 Subject: [PATCH 2/6] feat(pieces): add RingCentral piece (#14668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR was approved, but it hasn’t been tested yet because the Pieces Policy isn’t running in our regions. It will be retested later. Co-authored-by: Kishan Parmar <135701940+kishanprmr@users.noreply.github.com> Co-authored-by: Odai Ahmad --- .../community/ringcentral/.eslintrc.json | 47 +++ .../pieces/community/ringcentral/README.md | 90 ++++++ .../pieces/community/ringcentral/package.json | 22 ++ .../ringcentral/src/i18n/translation.json | 88 ++++++ .../community/ringcentral/src/index.test.ts | 60 ++++ .../pieces/community/ringcentral/src/index.ts | 45 +++ .../src/lib/actions/actions.test.ts | 272 ++++++++++++++++++ .../actions/download-message-attachment.ts | 98 +++++++ .../src/lib/actions/get-call-log.ts | 67 +++++ .../src/lib/actions/get-extension-info.ts | 19 ++ .../src/lib/actions/get-message.ts | 34 +++ .../ringcentral/src/lib/actions/make-call.ts | 52 ++++ .../ringcentral/src/lib/actions/send-sms.ts | 39 +++ .../src/lib/actions/send-team-message.ts | 30 ++ .../ringcentral/src/lib/common/auth.ts | 35 +++ .../ringcentral/src/lib/common/client.test.ts | 149 ++++++++++ .../ringcentral/src/lib/common/client.ts | 176 ++++++++++++ .../ringcentral/src/lib/common/props.test.ts | 118 ++++++++ .../ringcentral/src/lib/common/props.ts | 146 ++++++++++ .../lib/common/subscription-trigger.test.ts | 137 +++++++++ .../src/lib/common/subscription-trigger.ts | 104 +++++++ .../src/lib/common/test-support/http-stub.ts | 99 +++++++ .../src/lib/triggers/new-inbound-sms.ts | 42 +++ .../src/lib/triggers/new-team-message.ts | 24 ++ .../src/lib/triggers/new-voicemail.ts | 31 ++ .../src/lib/triggers/triggers.test.ts | 88 ++++++ .../community/ringcentral/tsconfig.json | 16 ++ .../community/ringcentral/tsconfig.lib.json | 20 ++ .../community/ringcentral/vitest.config.ts | 18 ++ 29 files changed, 2166 insertions(+) create mode 100644 packages/pieces/community/ringcentral/.eslintrc.json create mode 100644 packages/pieces/community/ringcentral/README.md create mode 100644 packages/pieces/community/ringcentral/package.json create mode 100644 packages/pieces/community/ringcentral/src/i18n/translation.json create mode 100644 packages/pieces/community/ringcentral/src/index.test.ts create mode 100644 packages/pieces/community/ringcentral/src/index.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/actions.test.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/download-message-attachment.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/get-call-log.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/get-extension-info.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/get-message.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/make-call.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/send-sms.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/actions/send-team-message.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/auth.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/client.test.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/client.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/props.test.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/props.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.test.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/common/test-support/http-stub.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/triggers/new-inbound-sms.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/triggers/new-team-message.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/triggers/new-voicemail.ts create mode 100644 packages/pieces/community/ringcentral/src/lib/triggers/triggers.test.ts create mode 100644 packages/pieces/community/ringcentral/tsconfig.json create mode 100644 packages/pieces/community/ringcentral/tsconfig.lib.json create mode 100644 packages/pieces/community/ringcentral/vitest.config.ts diff --git a/packages/pieces/community/ringcentral/.eslintrc.json b/packages/pieces/community/ringcentral/.eslintrc.json new file mode 100644 index 000000000000..6f1536634f91 --- /dev/null +++ b/packages/pieces/community/ringcentral/.eslintrc.json @@ -0,0 +1,47 @@ +{ + "extends": [ + "../../../../.eslintrc.json" + ], + "ignorePatterns": [ + "!**/*" + ], + "overrides": [ + { + "files": [ + "*.ts", + "*.tsx", + "*.js", + "*.jsx" + ], + "rules": {} + }, + { + "files": [ + "*.ts", + "*.tsx" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + "lodash", + "lodash/*", + "@activepieces/core-*", + "@activepieces/server*", + "@activepieces/engine", + "@activepieces/shared" + ] + } + ] + } + }, + { + "files": [ + "*.js", + "*.jsx" + ], + "rules": {} + } + ] +} diff --git a/packages/pieces/community/ringcentral/README.md b/packages/pieces/community/ringcentral/README.md new file mode 100644 index 000000000000..10419e0be1be --- /dev/null +++ b/packages/pieces/community/ringcentral/README.md @@ -0,0 +1,90 @@ +# RingCentral piece (`@activepieces/piece-ringcentral`) + +SMS, RingOut calls, call logs and Team Messaging. + +## Connection setup + +In the [RingCentral Developer Console](https://developers.ringcentral.com/), create a REST API app +using **OAuth 2.0 Authorization Code Flow** for a server/web app: + +1. Add the redirect URI shown on the connection dialog to the app. +2. Enable the app scopes the flow needs: SMS, RingOut, Read Messages, Read Call Log, Read Accounts, + TeamMessaging, Webhook Subscriptions. +3. Paste the app's Client ID and Client Secret into the connection, and pick the Environment + (Production or Sandbox) the app is registered on. Sandbox apps only work against + `platform.devtest.ringcentral.com`; graduation to production is a RingCentral-side step. + +Every connection is an ordinary per-user OAuth login. Nothing here is shared platform-wide, which is +also why this piece keeps a Custom API Call action: it can only do what the connection's owner +already can. + +## Actions + +| Action | Endpoint | Notes | +|---|---|---| +| Send SMS | `POST /restapi/v1.0/account/~/extension/~/sms` | From is a dropdown of the extension's SMS-enabled numbers | +| Make Call (RingOut) | `POST /restapi/v1.0/account/~/extension/~/ring-out` | Two-legged: calls "from" first, then connects "to" | +| Send Team Messaging Post | `POST /team-messaging/v1/chats/{chatId}/posts` | Markdown supported; Chat is a dropdown | +| Get Call Log | `GET /restapi/v1.0/account/~/extension/~/call-log` | Direction/type/date filters, paging via perPage | +| Get Extension Info | `GET /restapi/v1.0/account/~/extension/~` | The authenticated extension's profile | +| Get Message | `GET /restapi/v1.0/account/~/extension/~/message-store/{messageId}` | Reads a text or voicemail back, including its attachment list | +| Download Message Attachment | `GET .../message-store/{messageId}/content/{attachmentId}` | Returns a file. Resolves the attachment id and name from the message when not given | +| Custom API Call | any | Bearer token of this connection | + +Reads retry on 5xx; writes never do, because a replayed RingOut dials someone twice and a replayed +SMS sends twice. Every request carries a 30s timeout. + +**The From dropdown lists only numbers carrying the `SmsSender` feature.** RingCentral assigns some +numbers to an extension for caller ID only; those come back with `features: ['CallerId']` and are +refused at send time with `MSG-242 FeatureNotAvailable`. Filtering the list is what keeps that from +being a run-time surprise. If the dropdown reports no numbers, none on the extension are SMS-enabled. + +The Chat dropdown follows RingCentral's page tokens rather than reading the first page only, so a +chat past the first 250 is still selectable. + +## Triggers + +All three are WebHook subscriptions (`/restapi/v1.0/subscription`) built by one factory +(`src/lib/common/subscription-trigger.ts`): + +| Trigger | Event filter | Kept deliveries | +|---|---|---| +| New Inbound SMS or MMS | `message-store/instant?type=SMS` | `direction === 'Inbound'` | +| New Voicemail | `voicemail` | all | +| New Team Messaging Post | `glip/posts` | `eventType === 'PostAdded'` | + +One filter on the text trigger, and `type=SMS` is right for picture messages too. There is no MMS +message type: RingCentral delivers an inbound MMS through the same filter with `type: 'SMS'` and an +extra `MmsAttachment` part. Do not add `type=MMS`, an unrecognised type can fail the whole +`createSubscription` call. Pair the trigger with **Download Message Attachment** to pull the media, +since the delivery carries only attachment metadata, never the bytes. + +Voicemail uses its own event filter (`.../extension/~/voicemail`), not `message-store/instant`, which +is documented for inbound SMS only. + +Behaviour worth knowing: + +- **Handshake:** RingCentral validates the endpoint by demanding its `Validation-Token` header + echoed back; the trigger answers via `onHandshake` + `WebhookHandshakeStrategy.HEADER_PRESENT`. +- **Deliveries are not signed.** The only secret a genuine delivery carries is the subscription id + minted at enable time, so `run()` compares `subscriptionId` against the stored one and drops + everything else. A fabricated POST to the webhook URL therefore does nothing. +- **Dedupe:** message/post id (falling back to the delivery uuid) becomes the platform dedupe key. + An event with no id at all passes through un-keyed rather than sharing a constant key, which + would silently swallow every later one as a duplicate. +- **Lifetime:** subscriptions are created with the documented 20-year maximum, but RingCentral + blacklists a subscription whose endpoint keeps failing deliveries, so disable/enable of the flow + re-mints it and `onDisable` tolerates an already-dead id. + +## Tests + +| File | Covers | +|---|---| +| `src/index.test.ts` | piece surface: auth wiring, action/trigger names | +| `src/lib/common/client.test.ts` | server selection, timeout/retry policy, error translation | +| `src/lib/common/subscription-trigger.test.ts` | handshake, lifecycle, subscriptionId filtering, dedupe | +| `src/lib/actions/actions.test.ts` | prop-to-request mapping per action, attachment resolution and download | +| `src/lib/common/props.test.ts` | dropdown option building: SMS capability filter, chat paging | +| `src/lib/triggers/triggers.test.ts` | the event filters each trigger subscribes to, inbound filtering | + +Run with `bun run test` from this directory. diff --git a/packages/pieces/community/ringcentral/package.json b/packages/pieces/community/ringcentral/package.json new file mode 100644 index 000000000000..b10ca30589a2 --- /dev/null +++ b/packages/pieces/community/ringcentral/package.json @@ -0,0 +1,22 @@ +{ + "name": "@activepieces/piece-ringcentral", + "version": "0.0.1", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "dependencies": { + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*" + }, + "devDependencies": { + "vitest": "3.2.6", + "tslib": "2.6.2" + }, + "scripts": { + "build": "tsc -p tsconfig.lib.json && cp package.json dist/", + "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle", + "lint": "eslint 'src/**/*.ts'", + "test": "vitest run" + } +} diff --git a/packages/pieces/community/ringcentral/src/i18n/translation.json b/packages/pieces/community/ringcentral/src/i18n/translation.json new file mode 100644 index 000000000000..1c300907bfc3 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/i18n/translation.json @@ -0,0 +1,88 @@ +{ + "Cloud business communications: send SMS, place calls, manage messages, and post to Team Messaging.": "Cloud business communications: send SMS, place calls, manage messages, and post to Team Messaging.", + "In the [RingCentral Developer Console](https://developers.ringcentral.com/), create a REST API app using **OAuth 2.0 (Authorization Code Flow)** for a server/web app. Add the redirect URI shown here to the app, then enable the app scopes you need (for example: SMS, RingOut, Read Messages, Read Call Log, Read Accounts, TeamMessaging, Webhook Subscriptions). Copy the app's Client ID and Client Secret below, and pick the Environment that matches your app.": "In the [RingCentral Developer Console](https://developers.ringcentral.com/), create a REST API app using **OAuth 2.0 (Authorization Code Flow)** for a server/web app. Add the redirect URI shown here to the app, then enable the app scopes you need (for example: SMS, RingOut, Read Messages, Read Call Log, Read Accounts, TeamMessaging, Webhook Subscriptions). Copy the app's Client ID and Client Secret below, and pick the Environment that matches your app.", + "Environment": "Environment", + "The RingCentral server your app is registered on.": "The RingCentral server your app is registered on.", + "Production": "Production", + "Sandbox": "Sandbox", + "Connection": "Connection", + "Send SMS": "Send SMS", + "Send an SMS text message from one of your RingCentral numbers.": "Send an SMS text message from one of your RingCentral numbers.", + "From": "From", + "The SMS-enabled RingCentral number to send from.": "The SMS-enabled RingCentral number to send from.", + "To": "To", + "Recipient phone number(s) in E.164 format (e.g. +14155550123).": "Recipient phone number(s) in E.164 format (e.g. +14155550123).", + "Message": "Message", + "The text content of the SMS message.": "The text content of the SMS message.", + "Make Call (RingOut)": "Make Call (RingOut)", + "Start a two-legged RingOut call: RingCentral calls the \"from\" number first, then connects it to the \"to\" number.": "Start a two-legged RingOut call: RingCentral calls the \"from\" number first, then connects it to the \"to\" number.", + "The number RingCentral calls first, in E.164 format (e.g. +14155550100). Usually one of your RingCentral numbers.": "The number RingCentral calls first, in E.164 format (e.g. +14155550100). Usually one of your RingCentral numbers.", + "The number to connect the call to, in E.164 format (e.g. +14155550123).": "The number to connect the call to, in E.164 format (e.g. +14155550123).", + "Caller ID": "Caller ID", + "Optional number shown to the callee, in E.164 format. Must be one of your RingCentral numbers.": "Optional number shown to the callee, in E.164 format. Must be one of your RingCentral numbers.", + "Play Prompt": "Play Prompt", + "Play a \"please hold\" prompt to the \"from\" party before connecting the call.": "Play a \"please hold\" prompt to the \"from\" party before connecting the call.", + "Send Team Messaging Post": "Send Team Messaging Post", + "Post a message to a RingCentral Team Messaging chat, group, or team.": "Post a message to a RingCentral Team Messaging chat, group, or team.", + "Chat": "Chat", + "The direct message, group or team to post to.": "The direct message, group or team to post to.", + "The text content of the post. Supports Markdown.": "The text content of the post. Supports Markdown.", + "Get Call Log": "Get Call Log", + "Retrieve call log records for the authenticated user's extension.": "Retrieve call log records for the authenticated user's extension.", + "Direction": "Direction", + "Filter records by call direction.": "Filter records by call direction.", + "Inbound": "Inbound", + "Outbound": "Outbound", + "Type": "Type", + "Filter records by call type.": "Filter records by call type.", + "Voice": "Voice", + "Fax": "Fax", + "Date From": "Date From", + "The start of the time range in ISO 8601 format (e.g. 2024-01-01T00:00:00Z).": "The start of the time range in ISO 8601 format (e.g. 2024-01-01T00:00:00Z).", + "Date To": "Date To", + "The end of the time range in ISO 8601 format (e.g. 2024-01-31T23:59:59Z).": "The end of the time range in ISO 8601 format (e.g. 2024-01-31T23:59:59Z).", + "Records Per Page": "Records Per Page", + "Maximum number of records to return (1-1000).": "Maximum number of records to return (1-1000).", + "Get Extension Info": "Get Extension Info", + "Get profile information for the authenticated user's extension.": "Get profile information for the authenticated user's extension.", + "Get Message": "Get Message", + "Retrieve a single SMS, MMS or voicemail message, including the list of attachments it carries.": "Retrieve a single SMS, MMS or voicemail message, including the list of attachments it carries.", + "Message ID": "Message ID", + "The message to read. The New Inbound SMS or MMS trigger emits this as `id`.": "The message to read. The New Inbound SMS or MMS trigger emits this as `id`.", + "Download Message Attachment": "Download Message Attachment", + "Download the content of a message attachment, such as an MMS photo or a voicemail recording, and return it as a file.": "Download the content of a message attachment, such as an MMS photo or a voicemail recording, and return it as a file.", + "The message the attachment belongs to.": "The message the attachment belongs to.", + "Attachment ID": "Attachment ID", + "Which attachment to download. Leave blank to take the first one that is not the message text, which is the usual case for an MMS carrying a single photo.": "Which attachment to download. Leave blank to take the first one that is not the message text, which is the usual case for an MMS carrying a single photo.", + "File Name": "File Name", + "Overrides the name RingCentral reports. Worth setting when the name matters downstream, e.g. naming a POD after the load number.": "Overrides the name RingCentral reports. Worth setting when the name matters downstream, e.g. naming a POD after the load number.", + "Custom API Call": "Custom API Call", + "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint", + "Method": "Method", + "GET": "GET", + "POST": "POST", + "PATCH": "PATCH", + "PUT": "PUT", + "DELETE": "DELETE", + "HEAD": "HEAD", + "Headers": "Headers", + "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.", + "Query Parameters": "Query Parameters", + "Body Type": "Body Type", + "None": "None", + "JSON": "JSON", + "Form Data": "Form Data", + "Raw": "Raw", + "Body": "Body", + "Response is Binary ?": "Response is Binary ?", + "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.", + "No Error on Failure": "No Error on Failure", + "Timeout (in seconds)": "Timeout (in seconds)", + "Follow redirects": "Follow redirects", + "New Inbound SMS or MMS": "New Inbound SMS or MMS", + "Triggers when a new inbound text message is received, with or without media attached.": "Triggers when a new inbound text message is received, with or without media attached.", + "New Voicemail": "New Voicemail", + "Triggers when a new voicemail message is received.": "Triggers when a new voicemail message is received.", + "New Team Messaging Post": "New Team Messaging Post", + "Triggers when a new post is added in RingCentral Team Messaging.": "Triggers when a new post is added in RingCentral Team Messaging." +} \ No newline at end of file diff --git a/packages/pieces/community/ringcentral/src/index.test.ts b/packages/pieces/community/ringcentral/src/index.test.ts new file mode 100644 index 000000000000..fc7f1b9a9357 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/index.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; + +import { OAuth2AuthorizationMethod } from '@activepieces/pieces-framework'; + +import { ringcentral } from './index'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const meta = (): any => ringcentral.metadata(); + +describe('piece metadata', () => { + it('declares the expected surface', () => { + const m = meta(); + expect(m.displayName).toBe('RingCentral'); + expect(Object.keys(m.actions)).toHaveLength(8); + expect(Object.keys(m.triggers)).toHaveLength(3); + }); + + it('exposes every action and trigger by name', () => { + expect(Object.keys(meta().actions).sort()).toEqual([ + 'custom_api_call', + 'download_message_attachment', + 'get_call_log', + 'get_extension_info', + 'get_message', + 'make_call', + 'send_sms', + 'send_team_message', + ]); + expect(Object.keys(meta().triggers).sort()).toEqual([ + 'new_inbound_sms', + 'new_team_message', + 'new_voicemail', + ]); + }); + + it('points the logo at the pieces CDN', () => { + expect(meta().logoUrl).toBe('https://cdn.activepieces.com/pieces/ringcentral.png'); + }); + + it('authenticates the token exchange with a Basic header', () => { + // The framework defaults to client creds in the request body, which RingCentral's token endpoint + // answers with OAU-123 "Client authentication is required" before it even looks at the grant. + // Refresh reads the same stored method, so BODY breaks reconnects too. + expect(meta().auth.authorizationMethod).toBe(OAuth2AuthorizationMethod.HEADER); + }); + + it('opts out of the platform-appended consent prompt', () => { + // With prompt=consent no connection can be created at all: RingCentral's login goes SSO-only. + expect(meta().auth.prompt).toBe('omit'); + }); + + it('lists its authors', () => { + expect(meta().authors).toEqual(['alexandronic']); + }); + + it('declares a minimum supported release', () => { + // createPiece may clamp the declared floor upward, so assert shape, not the exact value. + expect(meta().minimumSupportedRelease).toMatch(/^\d+\.\d+\.\d+$/); + }); +}); diff --git a/packages/pieces/community/ringcentral/src/index.ts b/packages/pieces/community/ringcentral/src/index.ts new file mode 100644 index 000000000000..ceb31fdffe8a --- /dev/null +++ b/packages/pieces/community/ringcentral/src/index.ts @@ -0,0 +1,45 @@ +import { createPiece, OAuth2PropertyValue, PieceCategory } from '@activepieces/pieces-framework'; +import { createCustomApiCallAction } from '@activepieces/pieces-common'; + +import { ringcentralAuth } from './lib/common/auth'; +import { ringcentralCommon } from './lib/common/client'; + +import { sendSms } from './lib/actions/send-sms'; +import { makeCall } from './lib/actions/make-call'; +import { sendTeamMessage } from './lib/actions/send-team-message'; +import { getCallLog } from './lib/actions/get-call-log'; +import { getExtensionInfo } from './lib/actions/get-extension-info'; +import { getMessage } from './lib/actions/get-message'; +import { downloadMessageAttachment } from './lib/actions/download-message-attachment'; +import { newInboundSms } from './lib/triggers/new-inbound-sms'; +import { newVoicemail } from './lib/triggers/new-voicemail'; +import { newTeamMessage } from './lib/triggers/new-team-message'; + +export const ringcentral = createPiece({ + displayName: 'RingCentral', + description: + 'Cloud business communications: send SMS, place calls, manage messages, and post to Team Messaging.', + auth: ringcentralAuth, + minimumSupportedRelease: '0.36.1', + logoUrl: 'https://cdn.activepieces.com/pieces/ringcentral.png', + authors: ['alexandronic'], + categories: [PieceCategory.COMMUNICATION], + actions: [ + sendSms, + makeCall, + sendTeamMessage, + getCallLog, + getExtensionInfo, + getMessage, + downloadMessageAttachment, + createCustomApiCallAction({ + auth: ringcentralAuth, + baseUrl: (auth) => + auth ? ringcentralCommon.getServerUrl(auth as OAuth2PropertyValue) : '', + authMapping: async (auth) => ({ + Authorization: `Bearer ${(auth as OAuth2PropertyValue).access_token}`, + }), + }), + ], + triggers: [newInboundSms, newVoicemail, newTeamMessage], +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/actions.test.ts b/packages/pieces/community/ringcentral/src/lib/actions/actions.test.ts new file mode 100644 index 000000000000..e355a47d2e23 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/actions.test.ts @@ -0,0 +1,272 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { httpError, memFiles, oauth, stubHttp } from '../common/test-support/http-stub'; +import { downloadMessageAttachment } from './download-message-attachment'; +import { getCallLog } from './get-call-log'; +import { getMessage } from './get-message'; +import { getExtensionInfo } from './get-extension-info'; +import { makeCall } from './make-call'; +import { sendSms } from './send-sms'; +import { sendTeamMessage } from './send-team-message'; + +afterEach(() => vi.restoreAllMocks()); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const ctx = (propsValue: Record): any => ({ + auth: oauth(), + propsValue, +}); + +describe('send_sms', () => { + it('wraps the numbers the way the SMS endpoint demands', async () => { + const stub = stubHttp(); + stub.route('/sms', { id: 1 }); + + await sendSms.run( + ctx({ from: '+14155550100', to: ['+14155550123', 14155550124], text: 'hi' }), + ); + + const sent = stub.find('/sms'); + expect(sent?.method).toBe('POST'); + expect(sent?.body).toEqual({ + from: { phoneNumber: '+14155550100' }, + // Numbers typed into an Array prop can arrive as non-strings; the wire wants strings. + to: [{ phoneNumber: '+14155550123' }, { phoneNumber: '14155550124' }], + text: 'hi', + }); + }); + + it('surfaces the translated error when the number cannot send SMS', async () => { + const stub = stubHttp(); + stub.route('/sms', () => + httpError(400, { errors: [{ errorCode: 'MSG-347', message: 'not SMS enabled' }] }), + ); + + await expect( + sendSms.run(ctx({ from: '+1', to: ['+2'], text: 'hi' })), + ).rejects.toThrow(/MSG-347/); + }); +}); + +describe('make_call', () => { + it('omits callerId unless one was given', async () => { + const stub = stubHttp(); + stub.route('/ring-out', { id: 1 }); + + await makeCall.run(ctx({ from: '+1', to: '+2' })); + + expect(stub.find('/ring-out')?.body).toEqual({ + from: { phoneNumber: '+1' }, + to: { phoneNumber: '+2' }, + playPrompt: false, + }); + }); + + it('passes callerId and playPrompt through when set', async () => { + const stub = stubHttp(); + stub.route('/ring-out', { id: 1 }); + + await makeCall.run(ctx({ from: '+1', to: '+2', callerId: '+3', playPrompt: true })); + + expect(stub.find('/ring-out')?.body).toMatchObject({ + callerId: { phoneNumber: '+3' }, + playPrompt: true, + }); + }); +}); + +describe('send_team_message', () => { + it('path-encodes the chat id', async () => { + const stub = stubHttp(); + stub.route('/team-messaging', { id: 'p1' }); + + await sendTeamMessage.run(ctx({ chatId: 'team/1', text: '**done**' })); + + const sent = stub.find('/team-messaging'); + expect(sent?.url).toMatch(/\/chats\/team%2F1\/posts$/); + expect(sent?.body).toEqual({ text: '**done**' }); + }); +}); + +describe('get_call_log', () => { + it('sends only the filters that were set, with perPage stringified', async () => { + const stub = stubHttp(); + stub.route('/call-log', { records: [] }); + + await getCallLog.run(ctx({ direction: 'Inbound', perPage: 50 })); + + expect(stub.find('/call-log')?.queryParams).toEqual({ + direction: 'Inbound', + perPage: '50', + }); + }); + + it('sends no query params when nothing was filtered', async () => { + const stub = stubHttp(); + stub.route('/call-log', { records: [] }); + + await getCallLog.run(ctx({})); + + expect(stub.find('/call-log')?.queryParams).toEqual({}); + }); +}); + +describe('get_extension_info', () => { + it('reads the authenticated extension', async () => { + const stub = stubHttp(); + stub.route('/extension/~', { name: 'Dispatch' }); + + const out = await getExtensionInfo.run(ctx({})); + + expect(out).toEqual({ name: 'Dispatch' }); + expect(stub.find('/extension/~')?.method).toBe('GET'); + }); +}); + +const MESSAGE_ID = '1234567890'; +const MESSAGE_PATH = `/message-store/${MESSAGE_ID}`; +const CONTENT_PATH = `/message-store/${MESSAGE_ID}/content/`; + +/** + * An MMS as RingCentral actually reports it: type is 'SMS' (there is no MMS type), and the media + * arrives as an extra MmsAttachment part alongside the Text part. + */ +const mmsWithPhoto = (overrides: Record = {}) => ({ + id: Number(MESSAGE_ID), + type: 'SMS', + direction: 'Inbound', + attachments: [ + { id: 111, type: 'Text', contentType: 'text/plain' }, + { id: 222, type: 'MmsAttachment', contentType: 'image/jpeg', fileName: 'pod.jpg' }, + ], + ...overrides, +}); + +describe('get_message', () => { + it('reads the message back by id', async () => { + const stub = stubHttp(); + stub.route(MESSAGE_PATH, mmsWithPhoto()); + + const result = await getMessage.run(ctx({ messageId: MESSAGE_ID })); + + expect(stub.find(MESSAGE_PATH)?.method).toBe('GET'); + expect(result).toMatchObject({ id: Number(MESSAGE_ID), type: 'SMS' }); + }); + + it('escapes the id rather than interpolating it into the path raw', async () => { + const stub = stubHttp(); + stub.route('/message-store/', { id: 'x' }); + + await getMessage.run(ctx({ messageId: 'a/../b' })); + + expect(stub.find('/message-store/')?.url).toContain('a%2F..%2Fb'); + }); +}); + +describe('download_message_attachment', () => { + /** + * The stub matches routes by substring, first registered wins, and the content URL contains the + * message URL. So the content route has to be registered first or the metadata JSON would answer + * the binary request too. + */ + function stubMessage(message: Record, content = Buffer.from('JPEGBYTES')) { + const stub = stubHttp(); + stub.route(CONTENT_PATH, content); + stub.route(MESSAGE_PATH, message); + return stub; + } + + function download(propsValue: Record) { + const files = memFiles(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const run = downloadMessageAttachment.run({ auth: oauth(), propsValue, files } as any); + return { files, run }; + } + + it('picks the media part by default, skipping the SMS text part', async () => { + const stub = stubMessage(mmsWithPhoto()); + + const { files, run } = download({ messageId: MESSAGE_ID }); + const result = await run; + + // 222 is the photo; 111 is the text body, which is already on the trigger payload. + expect(stub.find(CONTENT_PATH)?.url).toContain('/content/222'); + expect(files.written[0].fileName).toBe('pod.jpg'); + expect(result.file).toBe('mock://files/pod.jpg'); + }); + + it('asks for the body as binary, so the bytes are not mangled into a string', async () => { + const stub = stubMessage(mmsWithPhoto()); + + await download({ messageId: MESSAGE_ID }).run; + + expect(stub.find(CONTENT_PATH)?.responseType).toBe('arraybuffer'); + // The metadata read stays JSON. + expect(stub.find(MESSAGE_PATH)?.responseType).toBeUndefined(); + }); + + it('honours an explicit attachment id', async () => { + const stub = stubMessage(mmsWithPhoto()); + + await download({ messageId: MESSAGE_ID, attachmentId: '111' }).run; + + expect(stub.find(CONTENT_PATH)?.url).toContain('/content/111'); + }); + + it('lets the caller name the file, for a POD named after its load', async () => { + stubMessage(mmsWithPhoto()); + + const { files, run } = download({ messageId: MESSAGE_ID, fileName: 'L-4471-pod.jpg' }); + await run; + + // Overrides the reported pod.jpg, because the name matters to whatever stores it next. + expect(files.written[0].fileName).toBe('L-4471-pod.jpg'); + }); + + it('reads the other spelling of the reported name', async () => { + stubMessage( + mmsWithPhoto({ + attachments: [{ id: 222, type: 'MmsAttachment', filename: 'lowercase.png' }], + }), + ); + + const { files, run } = download({ messageId: MESSAGE_ID }); + await run; + + expect(files.written[0].fileName).toBe('lowercase.png'); + }); + + it('falls back to a generated name when RingCentral reports none', async () => { + stubMessage(mmsWithPhoto({ attachments: [{ id: 222, type: 'MmsAttachment' }] })); + + const { files, run } = download({ messageId: MESSAGE_ID }); + await run; + + expect(files.written[0].fileName).toBe(`ringcentral-${MESSAGE_ID}-222`); + }); + + it('explains an SMS that carries nothing to download', async () => { + stubMessage({ id: Number(MESSAGE_ID), type: 'SMS', attachments: [] }); + + await expect(download({ messageId: MESSAGE_ID }).run).rejects.toThrow( + /carries no attachments/, + ); + }); + + it('lists what is actually there when the given attachment id is wrong', async () => { + stubMessage(mmsWithPhoto()); + + await expect( + download({ messageId: MESSAGE_ID, attachmentId: '999' }).run, + ).rejects.toThrow(/has no attachment 999.*222/s); + }); + + it('never fetches content when the metadata read fails', async () => { + const stub = stubHttp(); + stub.route(CONTENT_PATH, Buffer.from('never')); + stub.route(MESSAGE_PATH, () => httpError(404, { message: 'Message not found' })); + + await expect(download({ messageId: MESSAGE_ID }).run).rejects.toThrow(/404/); + expect(stub.find('/content/')).toBeUndefined(); + }); +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/download-message-attachment.ts b/packages/pieces/community/ringcentral/src/lib/actions/download-message-attachment.ts new file mode 100644 index 000000000000..ec7963aa2865 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/download-message-attachment.ts @@ -0,0 +1,98 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; + +export const downloadMessageAttachment = createAction({ + auth: ringcentralAuth, + name: 'download_message_attachment', + displayName: 'Download Message Attachment', + description: + 'Download the content of a message attachment, such as an MMS photo or a voicemail recording, and return it as a file.', + props: { + messageId: Property.ShortText({ + displayName: 'Message ID', + description: 'The message the attachment belongs to.', + required: true, + }), + attachmentId: Property.ShortText({ + displayName: 'Attachment ID', + description: + 'Which attachment to download. Leave blank to take the first one that is not the message text, which is the usual case for an MMS carrying a single photo.', + required: false, + }), + fileName: Property.ShortText({ + displayName: 'File Name', + description: + "Overrides the name RingCentral reports. Worth setting when the name matters downstream, e.g. naming a POD after the load number.", + required: false, + }), + }, + async run(context) { + const { messageId, attachmentId, fileName } = context.propsValue; + + // Read the message first, the same shape as other pieces' download actions: it resolves the + // attachment id when the caller did not supply one, and gives the reported file name. It also + // turns a wrong id into a clear message rather than a bare 404 from the content endpoint. + const message = await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.GET, + resourcePath: `/restapi/v1.0/account/~/extension/~/message-store/${encodeURIComponent( + messageId, + )}`, + }); + + const attachments = message.attachments ?? []; + if (attachments.length === 0) { + throw new Error( + `Message ${messageId} carries no attachments. An SMS with no media has none, so check the message id or use Get Message to inspect it.`, + ); + } + + const attachment = attachmentId + ? attachments.find((a) => String(a.id) === String(attachmentId)) + : // 'Text' is the SMS body itself, which is already on the trigger payload and is not the file + // anyone means by "the attachment", so it is skipped when picking a default. + (attachments.find((a) => a.type !== 'Text') ?? attachments[0]); + + if (!attachment) { + throw new Error( + `Message ${messageId} has no attachment ${attachmentId}. It carries: ${attachments + .map((a) => `${a.id} (${a.type ?? 'unknown type'})`) + .join(', ')}.`, + ); + } + + const content = await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.GET, + resourcePath: `/restapi/v1.0/account/~/extension/~/message-store/${encodeURIComponent( + messageId, + )}/content/${encodeURIComponent(String(attachment.id))}`, + responseType: 'arraybuffer', + }); + + const reportedName = attachment.fileName ?? attachment.filename; + const file = await context.files.write({ + fileName: fileName ?? reportedName ?? `ringcentral-${messageId}-${attachment.id}`, + data: Buffer.from(content), + }); + + return { file, attachment }; + }, +}); + +type MessageAttachment = { + id?: string | number; + type?: string; + contentType?: string; + size?: number; + // RingCentral has shipped both spellings across API versions, so read either. + fileName?: string; + filename?: string; +}; + +type MessageWithAttachments = { + id?: string | number; + attachments?: MessageAttachment[]; +}; diff --git a/packages/pieces/community/ringcentral/src/lib/actions/get-call-log.ts b/packages/pieces/community/ringcentral/src/lib/actions/get-call-log.ts new file mode 100644 index 000000000000..07117a8dfcc1 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/get-call-log.ts @@ -0,0 +1,67 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod, QueryParams } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; + +export const getCallLog = createAction({ + auth: ringcentralAuth, + name: 'get_call_log', + displayName: 'Get Call Log', + description: "Retrieve call log records for the authenticated user's extension.", + props: { + direction: Property.StaticDropdown({ + displayName: 'Direction', + description: 'Filter records by call direction.', + required: false, + options: { + options: [ + { label: 'Inbound', value: 'Inbound' }, + { label: 'Outbound', value: 'Outbound' }, + ], + }, + }), + type: Property.StaticDropdown({ + displayName: 'Type', + description: 'Filter records by call type.', + required: false, + options: { + options: [ + { label: 'Voice', value: 'Voice' }, + { label: 'Fax', value: 'Fax' }, + ], + }, + }), + dateFrom: Property.ShortText({ + displayName: 'Date From', + description: 'The start of the time range in ISO 8601 format (e.g. 2024-01-01T00:00:00Z).', + required: false, + }), + dateTo: Property.ShortText({ + displayName: 'Date To', + description: 'The end of the time range in ISO 8601 format (e.g. 2024-01-31T23:59:59Z).', + required: false, + }), + perPage: Property.Number({ + displayName: 'Records Per Page', + description: 'Maximum number of records to return (1-1000).', + required: false, + }), + }, + async run(context) { + const { direction, type, dateFrom, dateTo, perPage } = context.propsValue; + + const queryParams: QueryParams = {}; + if (direction) queryParams['direction'] = direction; + if (type) queryParams['type'] = type; + if (dateFrom) queryParams['dateFrom'] = dateFrom; + if (dateTo) queryParams['dateTo'] = dateTo; + if (perPage) queryParams['perPage'] = String(perPage); + + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.GET, + resourcePath: '/restapi/v1.0/account/~/extension/~/call-log', + queryParams, + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/get-extension-info.ts b/packages/pieces/community/ringcentral/src/lib/actions/get-extension-info.ts new file mode 100644 index 000000000000..f18822fdfb76 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/get-extension-info.ts @@ -0,0 +1,19 @@ +import { createAction } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; + +export const getExtensionInfo = createAction({ + auth: ringcentralAuth, + name: 'get_extension_info', + displayName: 'Get Extension Info', + description: 'Get profile information for the authenticated user\'s extension.', + props: {}, + async run(context) { + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.GET, + resourcePath: '/restapi/v1.0/account/~/extension/~', + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/get-message.ts b/packages/pieces/community/ringcentral/src/lib/actions/get-message.ts new file mode 100644 index 000000000000..d05d03d4ca0f --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/get-message.ts @@ -0,0 +1,34 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; + +export const getMessage = createAction({ + auth: ringcentralAuth, + name: 'get_message', + displayName: 'Get Message', + description: + 'Retrieve a single SMS, MMS or voicemail message, including the list of attachments it carries.', + props: { + messageId: Property.ShortText({ + displayName: 'Message ID', + description: + 'The message to read. The New Inbound SMS or MMS trigger emits this as `id`.', + required: true, + }), + }, + async run(context) { + const { messageId } = context.propsValue; + + // The webhook delivery already carries most of this, but not reliably the attachment list for an + // MMS, and a flow that resumes from a stored id has nothing but the id. Reading the message back + // is also how you discover the attachment ids that Download Message Attachment needs. + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.GET, + resourcePath: `/restapi/v1.0/account/~/extension/~/message-store/${encodeURIComponent( + messageId, + )}`, + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/make-call.ts b/packages/pieces/community/ringcentral/src/lib/actions/make-call.ts new file mode 100644 index 000000000000..3e0bb66e9064 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/make-call.ts @@ -0,0 +1,52 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; + +export const makeCall = createAction({ + auth: ringcentralAuth, + name: 'make_call', + displayName: 'Make Call (RingOut)', + description: + 'Start a two-legged RingOut call: RingCentral calls the "from" number first, then connects it to the "to" number.', + props: { + from: Property.ShortText({ + displayName: 'From', + description: + 'The number RingCentral calls first, in E.164 format (e.g. +14155550100). Usually one of your RingCentral numbers.', + required: true, + }), + to: Property.ShortText({ + displayName: 'To', + description: 'The number to connect the call to, in E.164 format (e.g. +14155550123).', + required: true, + }), + callerId: Property.ShortText({ + displayName: 'Caller ID', + description: + 'Optional number shown to the callee, in E.164 format. Must be one of your RingCentral numbers.', + required: false, + }), + playPrompt: Property.Checkbox({ + displayName: 'Play Prompt', + description: 'Play a "please hold" prompt to the "from" party before connecting the call.', + required: false, + defaultValue: false, + }), + }, + async run(context) { + const { from, to, callerId, playPrompt } = context.propsValue; + + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.POST, + resourcePath: '/restapi/v1.0/account/~/extension/~/ring-out', + body: { + from: { phoneNumber: from }, + to: { phoneNumber: to }, + ...(callerId ? { callerId: { phoneNumber: callerId } } : {}), + playPrompt: playPrompt ?? false, + }, + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/send-sms.ts b/packages/pieces/community/ringcentral/src/lib/actions/send-sms.ts new file mode 100644 index 000000000000..f6103e289dcf --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/send-sms.ts @@ -0,0 +1,39 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; +import { smsFromNumberDropdown } from '../common/props'; + +export const sendSms = createAction({ + auth: ringcentralAuth, + name: 'send_sms', + displayName: 'Send SMS', + description: 'Send an SMS text message from one of your RingCentral numbers.', + props: { + from: smsFromNumberDropdown, + to: Property.Array({ + displayName: 'To', + description: 'Recipient phone number(s) in E.164 format (e.g. +14155550123).', + required: true, + }), + text: Property.LongText({ + displayName: 'Message', + description: 'The text content of the SMS message.', + required: true, + }), + }, + async run(context) { + const { from, to, text } = context.propsValue; + + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.POST, + resourcePath: '/restapi/v1.0/account/~/extension/~/sms', + body: { + from: { phoneNumber: from }, + to: to.map((phoneNumber) => ({ phoneNumber: String(phoneNumber) })), + text, + }, + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/actions/send-team-message.ts b/packages/pieces/community/ringcentral/src/lib/actions/send-team-message.ts new file mode 100644 index 000000000000..abb8a96c0dc5 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/actions/send-team-message.ts @@ -0,0 +1,30 @@ +import { createAction, Property } from '@activepieces/pieces-framework'; +import { HttpMethod } from '@activepieces/pieces-common'; +import { ringcentralAuth } from '../common/auth'; +import { ringcentralCommon } from '../common/client'; +import { chatDropdown } from '../common/props'; + +export const sendTeamMessage = createAction({ + auth: ringcentralAuth, + name: 'send_team_message', + displayName: 'Send Team Messaging Post', + description: 'Post a message to a RingCentral Team Messaging chat, group, or team.', + props: { + chatId: chatDropdown, + text: Property.LongText({ + displayName: 'Message', + description: 'The text content of the post. Supports Markdown.', + required: true, + }), + }, + async run(context) { + const { chatId, text } = context.propsValue; + + return await ringcentralCommon.sendRequest({ + auth: context.auth, + method: HttpMethod.POST, + resourcePath: `/team-messaging/v1/chats/${encodeURIComponent(chatId)}/posts`, + body: { text }, + }); + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/common/auth.ts b/packages/pieces/community/ringcentral/src/lib/common/auth.ts new file mode 100644 index 000000000000..192dcd5ef327 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/auth.ts @@ -0,0 +1,35 @@ +import { + OAuth2AuthorizationMethod, + PieceAuth, + Property, +} from '@activepieces/pieces-framework'; + +export const ringcentralAuth = PieceAuth.OAuth2({ + description: + "In the [RingCentral Developer Console](https://developers.ringcentral.com/), create a REST API app using **OAuth 2.0 (Authorization Code Flow)** for a server/web app. Add the redirect URI shown here to the app, then enable the app scopes you need (for example: SMS, RingOut, Read Messages, Read Call Log, Read Accounts, TeamMessaging, Webhook Subscriptions). Copy the app's Client ID and Client Secret below, and pick the Environment that matches your app.", + required: true, + props: { + environment: Property.StaticDropdown({ + displayName: 'Environment', + description: 'The RingCentral server your app is registered on.', + required: true, + defaultValue: 'platform.ringcentral.com', + options: { + disabled: false, + options: [ + { label: 'Production', value: 'platform.ringcentral.com' }, + { label: 'Sandbox', value: 'platform.devtest.ringcentral.com' }, + ], + }, + }), + }, + // RingCentral's token endpoint answers client creds in the request body with OAU-123 "Client + // authentication is required" before it looks at the grant, and the framework defaults to + // OAuth2AuthorizationMethod.BODY. Refresh reads the same stored method, so BODY breaks reconnects too. + authorizationMethod: OAuth2AuthorizationMethod.HEADER, + authUrl: 'https://{environment}/restapi/oauth/authorize', + tokenUrl: 'https://{environment}/restapi/oauth/token', + scope: [], + // The platform's default prompt=consent sends RingCentral's login to SSO-only (API_ERROR_208). + prompt: 'omit', +}); diff --git a/packages/pieces/community/ringcentral/src/lib/common/client.test.ts b/packages/pieces/community/ringcentral/src/lib/common/client.test.ts new file mode 100644 index 000000000000..2441f94c1c74 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/client.test.ts @@ -0,0 +1,149 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { describeRingCentralError, ringcentralCommon } from './client'; +import { httpError, oauth, stubHttp } from './test-support/http-stub'; + +afterEach(() => vi.restoreAllMocks()); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const auth = (overrides: Record = {}): any => oauth(overrides); + +describe('getServerUrl', () => { + it('uses the environment the connection picked', () => { + expect(ringcentralCommon.getServerUrl(auth())).toBe( + 'https://platform.devtest.ringcentral.com', + ); + }); + + it('falls back to production when the connection carries no environment', () => { + expect(ringcentralCommon.getServerUrl(auth({ props: undefined }))).toBe( + 'https://platform.ringcentral.com', + ); + }); +}); + +describe('sendRequest', () => { + it('sends the bearer token and a timeout on every call', async () => { + const stub = stubHttp(); + stub.route('/restapi/v1.0/account', { id: 42 }); + + const body = await ringcentralCommon.sendRequest({ + auth: auth(), + method: HttpMethod.GET, + resourcePath: '/restapi/v1.0/account/~/extension/~', + }); + + expect(body).toEqual({ id: 42 }); + const sent = stub.calls[0]; + expect(sent.url).toBe( + 'https://platform.devtest.ringcentral.com/restapi/v1.0/account/~/extension/~', + ); + expect(sent.authentication).toMatchObject({ token: 'RC_TOKEN' }); + expect(sent.timeout).toBeGreaterThan(0); + }); + + it('retries reads but never writes, because a replayed write sends twice', async () => { + const stub = stubHttp(); + stub.route('/call-log', {}); + stub.route('/sms', {}); + + await ringcentralCommon.sendRequest({ + auth: auth(), + method: HttpMethod.GET, + resourcePath: '/restapi/v1.0/account/~/extension/~/call-log', + }); + await ringcentralCommon.sendRequest({ + auth: auth(), + method: HttpMethod.POST, + resourcePath: '/restapi/v1.0/account/~/extension/~/sms', + }); + + expect(stub.find('/call-log')?.retries).toBeGreaterThan(0); + expect(stub.find('/sms')?.retries).toBe(0); + }); + + it('translates an HTTP failure instead of leaking a stringified axios error', async () => { + const stub = stubHttp(); + stub.route('/sms', () => httpError(403, { errorCode: 'InsufficientPermissions' })); + + await expect( + ringcentralCommon.sendRequest({ + auth: auth(), + method: HttpMethod.POST, + resourcePath: '/restapi/v1.0/account/~/extension/~/sms', + }), + ).rejects.toThrow(/Developer Console is missing the permission/); + }); +}); + +describe('subscriptions', () => { + it('creates a WebHook subscription with the filters and the flow webhook URL', async () => { + const stub = stubHttp(); + stub.route('/subscription', { id: 'sub-123' }); + + const id = await ringcentralCommon.createSubscription({ + auth: auth(), + webhookUrl: 'https://example.com/webhook/abc', + eventFilters: ['/restapi/v1.0/glip/posts'], + }); + + expect(id).toBe('sub-123'); + const sent = stub.find('/subscription'); + expect(sent?.method).toBe('POST'); + expect(sent?.body).toMatchObject({ + eventFilters: ['/restapi/v1.0/glip/posts'], + deliveryMode: { transportType: 'WebHook', address: 'https://example.com/webhook/abc' }, + }); + expect(sent?.body?.['expiresIn']).toBeGreaterThan(0); + }); + + it('deletes by id, path-encoded', async () => { + const stub = stubHttp(); + stub.route('/subscription/', {}); + + await ringcentralCommon.deleteSubscription({ auth: auth(), subscriptionId: 'sub/9?x' }); + + const sent = stub.find('/subscription/'); + expect(sent?.method).toBe('DELETE'); + expect(sent?.url).toMatch(/\/subscription\/sub%2F9%3Fx$/); + }); +}); + +describe('describeRingCentralError', () => { + const call = (err: unknown) => + describeRingCentralError(err, HttpMethod.POST, '/restapi/v1.0/account/~/extension/~/sms'); + + it('tells the user to reconnect on 401', () => { + expect(call(httpError(401, {}))).toMatch(/Reconnect the RingCentral connection/); + }); + + it('points at app permissions on 403 and carries RingCentral detail', () => { + const described = call( + httpError(403, { + errorCode: 'CMN-408', + message: 'In order to call this API endpoint, application needs to have [SMS] permission', + }), + ); + expect(described).toMatch(/missing the permission/); + expect(described).toMatch(/SMS/); + }); + + it('names the rate limit on 429', () => { + expect(call(httpError(429, {}))).toMatch(/rate-limited/); + }); + + it('surfaces the per-field errors array when present', () => { + const described = call( + httpError(400, { + errors: [{ errorCode: 'MSG-347', message: 'Phone number is not SMS enabled' }], + }), + ); + expect(described).toMatch(/MSG-347 Phone number is not SMS enabled/); + }); + + it('reports a transport failure as its own thing', () => { + expect(call(new Error('socket hang up'))).toMatch(/before a response arrived/); + expect(call(new Error('socket hang up'))).toMatch(/socket hang up/); + }); +}); diff --git a/packages/pieces/community/ringcentral/src/lib/common/client.ts b/packages/pieces/community/ringcentral/src/lib/common/client.ts new file mode 100644 index 000000000000..05501d7711eb --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/client.ts @@ -0,0 +1,176 @@ +import { + AuthenticationType, + HttpError, + HttpMethod, + HttpRequest, + QueryParams, + httpClient, +} from '@activepieces/pieces-common'; +import { OAuth2PropertyValue } from '@activepieces/pieces-framework'; + +export const ringcentralCommon = { + getServerUrl, + sendRequest, + createSubscription, + deleteSubscription, +}; + +/** + * RingCentral 4xx bodies name the real cause (`{ errorCode, message, errors: [...] }`), so surface + * that instead of a stringified axios failure. The bearer token travels in a header, never in the + * URL or body, so echoing the response body is safe here. + */ +export function describeRingCentralError( + err: unknown, + method: HttpMethod, + resourcePath: string, +): string { + const call = `${method} ${resourcePath}`; + if (!(err instanceof HttpError)) { + const detail = err instanceof Error && err.message ? ` (${err.message})` : ''; + return `RingCentral request ${call} failed before a response arrived${detail}.`; + } + + const { status, body } = err.errorMessage().response as { + status: number; + body?: RingCentralErrorBody; + }; + const detail = ringcentralErrorDetail(body); + + switch (status) { + case 401: + return `RingCentral rejected the connection's token (401 on ${call}). Reconnect the RingCentral connection.${detail}`; + case 403: + return `RingCentral refused ${call} (403). Usually the app in the RingCentral Developer Console is missing the permission this needs (for example SMS, RingOut or TeamMessaging).${detail}`; + case 429: + return `RingCentral rate-limited ${call} (429). Wait for the window to reset or slow the flow down.${detail}`; + default: + return `RingCentral answered ${status} for ${call}.${detail}`; + } +} + +const PRODUCTION_SERVER = 'platform.ringcentral.com'; + +// The documented maximum lifetime of a WebHook subscription (20 years). RingCentral still kills a +// subscription on its own when the endpoint keeps failing deliveries (blacklisting), which is why +// unsubscribe tolerates an already-dead id rather than assuming this expiry is ever reached. +const SUBSCRIPTION_EXPIRES_IN_SECONDS = 630720000; + +// RingCentral answers interactive calls well under a second; a step that sits longer than this is +// stuck, and a stuck step stalls the whole flow run. +const REQUEST_TIMEOUT_MS = 30_000; + +// 5xx-only, with backoff, per pieces-common. Reads are safe to repeat; writes are not retried at +// all because a replayed RingOut dials someone twice and a replayed SMS sends twice. +const READ_RETRIES = 3; + +// Declarations rather than arrow consts on purpose: they are hoisted, which is what lets +// `ringcentralCommon` above collect them before they appear in source order. +function getServerUrl(auth: OAuth2PropertyValue): string { + const server = auth.props?.['environment'] ?? PRODUCTION_SERVER; + return `https://${server}`; +} + +async function sendRequest({ + auth, + method, + resourcePath, + body, + queryParams, + responseType, +}: { + auth: OAuth2PropertyValue; + method: HttpMethod; + resourcePath: string; + body?: unknown; + queryParams?: QueryParams; + /** `arraybuffer` for the binary endpoints, e.g. message attachment content. Defaults to JSON. */ + responseType?: HttpRequest['responseType']; +}): Promise { + const request: HttpRequest = { + method, + url: `${getServerUrl(auth)}${resourcePath}`, + authentication: { + type: AuthenticationType.BEARER_TOKEN, + token: auth.access_token, + }, + body, + queryParams, + responseType, + timeout: REQUEST_TIMEOUT_MS, + retries: method === HttpMethod.GET ? READ_RETRIES : 0, + }; + + try { + const response = await httpClient.sendRequest(request); + return response.body; + } catch (err) { + throw new Error(describeRingCentralError(err, method, resourcePath)); + } +} + +async function createSubscription({ + auth, + webhookUrl, + eventFilters, +}: { + auth: OAuth2PropertyValue; + webhookUrl: string; + eventFilters: string[]; +}): Promise { + const subscription = await sendRequest<{ id: string }>({ + auth, + method: HttpMethod.POST, + resourcePath: '/restapi/v1.0/subscription', + body: { + eventFilters, + deliveryMode: { + transportType: 'WebHook', + address: webhookUrl, + }, + expiresIn: SUBSCRIPTION_EXPIRES_IN_SECONDS, + }, + }); + + return subscription.id; +} + +async function deleteSubscription({ + auth, + subscriptionId, +}: { + auth: OAuth2PropertyValue; + subscriptionId: string; +}): Promise { + await sendRequest({ + auth, + method: HttpMethod.DELETE, + resourcePath: `/restapi/v1.0/subscription/${encodeURIComponent(subscriptionId)}`, + }); +} + +function ringcentralErrorDetail(body: RingCentralErrorBody | undefined): string { + if (!body) return ''; + const messages = (body.errors ?? []) + .filter((e): e is { errorCode?: string; message?: string } => e !== null) + .map((e) => [e.errorCode, e.message].filter(Boolean).join(' ')); + if (messages.length === 0 && body.message) { + messages.push([body.errorCode, body.message].filter(Boolean).join(' ')); + } + return messages.length > 0 ? ` RingCentral says: ${messages.join('; ')}` : ''; +} + +type RingCentralErrorBody = { + errorCode?: string; + message?: string; + errors?: Array<{ errorCode?: string; message?: string } | null>; +}; + +export type RingCentralWebhookEvent> = { + uuid?: string; + event?: string; + timestamp?: string; + subscriptionId?: string; + ownerId?: string; + body?: T; +}; diff --git a/packages/pieces/community/ringcentral/src/lib/common/props.test.ts b/packages/pieces/community/ringcentral/src/lib/common/props.test.ts new file mode 100644 index 000000000000..ad9d9fb9a797 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/props.test.ts @@ -0,0 +1,118 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { chatDropdown, smsFromNumberDropdown } from './props'; +import { oauth, stubHttp } from './test-support/http-stub'; + +afterEach(() => vi.restoreAllMocks()); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const optionsOf = (prop: any, auth: unknown) => prop.options({ auth }, {} as any); + +const PHONE_PATH = '/extension/~/phone-number'; +const CHATS_PATH = '/team-messaging/v1/chats'; + +describe('smsFromNumberDropdown', () => { + it('offers only numbers carrying the SmsSender feature', async () => { + const stub = stubHttp(); + stub.route(PHONE_PATH, { + records: [ + { phoneNumber: '+14155550100', usageType: 'DirectNumber', features: ['SmsSender', 'CallerId'] }, + // Assigned for caller ID only. RingCentral refuses a send from it with MSG-242, so offering + // it would just move the failure to run time. + { phoneNumber: '+14155550111', usageType: 'DirectNumber', features: ['CallerId'] }, + { phoneNumber: '+14155550122', usageType: 'MainCompanyNumber' }, + ], + }); + + const result = await optionsOf(smsFromNumberDropdown, oauth()); + + expect(result.disabled).toBe(false); + expect(result.options).toEqual([ + { label: '+14155550100 (DirectNumber)', value: '+14155550100' }, + ]); + }); + + it('says why the list is empty rather than showing an empty dropdown', async () => { + const stub = stubHttp(); + stub.route(PHONE_PATH, { records: [{ phoneNumber: '+1', features: ['CallerId'] }] }); + + const result = await optionsOf(smsFromNumberDropdown, oauth()); + + expect(result.disabled).toBe(true); + expect(result.placeholder).toMatch(/No SMS-enabled numbers/); + }); + + it('asks for the numbers in one page of the documented maximum', async () => { + const stub = stubHttp(); + stub.route(PHONE_PATH, { records: [] }); + + await optionsOf(smsFromNumberDropdown, oauth()); + + expect(stub.find(PHONE_PATH)?.queryParams).toEqual({ perPage: '1000' }); + }); + + it('prompts to connect before an account exists', async () => { + const result = await optionsOf(smsFromNumberDropdown, undefined); + expect(result.disabled).toBe(true); + expect(result.options).toEqual([]); + }); + + it('tolerates a response with no records array', async () => { + const stub = stubHttp(); + stub.route(PHONE_PATH, {}); + const result = await optionsOf(smsFromNumberDropdown, oauth()); + expect(result.disabled).toBe(true); + }); +}); + +describe('chatDropdown', () => { + it('labels a named chat by name and an unnamed one by type and id', async () => { + const stub = stubHttp(); + stub.route(CHATS_PATH, { + records: [ + { id: '111', name: 'Dispatch', type: 'Team' }, + { id: '222', type: 'Direct' }, + ], + }); + + const result = await optionsOf(chatDropdown, oauth()); + + expect(result.options).toEqual([ + { label: 'Dispatch', value: '111' }, + { label: 'Direct (222)', value: '222' }, + ]); + }); + + it('follows the page tokens, so a chat past the first page is still selectable', async () => { + const stub = stubHttp(); + let call = 0; + stub.route(CHATS_PATH, () => { + call++; + return call === 1 + ? { records: [{ id: '1', name: 'first' }], navigation: { nextPageToken: 'tok-2' } } + : { records: [{ id: '2', name: 'second' }] }; + }); + + const result = await optionsOf(chatDropdown, oauth()); + + expect(result.options.map((o: { value: string }) => o.value)).toEqual(['1', '2']); + // The second request carries the token from the first. + expect(stub.calls.filter((c) => c.url.includes(CHATS_PATH))).toHaveLength(2); + expect(stub.calls[1].queryParams).toMatchObject({ pageToken: 'tok-2' }); + }); + + it('stops at the page cap rather than looping forever on a repeating token', async () => { + const stub = stubHttp(); + // A server that always reports another page would otherwise hang the dropdown. + stub.route(CHATS_PATH, { records: [{ id: 'x' }], navigation: { nextPageToken: 'same' } }); + + await optionsOf(chatDropdown, oauth()); + + expect(stub.calls.filter((c) => c.url.includes(CHATS_PATH))).toHaveLength(10); + }); + + it('prompts to connect before an account exists', async () => { + const result = await optionsOf(chatDropdown, undefined); + expect(result.disabled).toBe(true); + }); +}); diff --git a/packages/pieces/community/ringcentral/src/lib/common/props.ts b/packages/pieces/community/ringcentral/src/lib/common/props.ts new file mode 100644 index 000000000000..aa404283c02b --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/props.ts @@ -0,0 +1,146 @@ +import { HttpMethod } from '@activepieces/pieces-common'; +import { OAuth2PropertyValue, Property } from '@activepieces/pieces-framework'; + +import { ringcentralAuth } from './auth'; +import { ringcentralCommon } from './client'; + +/** + * The numbers this extension may actually send SMS from. + * + * RingCentral accepts a `from` number only when it is assigned to the extension behind the token AND + * carries the `SmsSender` feature. A number assigned for caller ID only comes back with + * `features: ['CallerId']` and is refused at send time with `MSG-242 FeatureNotAvailable`. On a real + * account most numbers are not SMS senders, so a free-text field means picking wrong is the default + * outcome and the error arrives only once the flow runs. + */ +export const smsFromNumberDropdown = Property.Dropdown({ + auth: ringcentralAuth, + displayName: 'From', + description: 'The SMS-enabled RingCentral number to send from.', + required: true, + refreshers: [], + options: async ({ auth }) => { + if (!auth) { + return notConnected(); + } + + const response = await ringcentralCommon.sendRequest>({ + auth: auth as OAuth2PropertyValue, + method: HttpMethod.GET, + resourcePath: '/restapi/v1.0/account/~/extension/~/phone-number', + // The documented maximum. An extension with more direct numbers than this is not a shape worth + // paging a dropdown for, and the SMS senders among them are a small subset anyway. + queryParams: { perPage: '1000' }, + }); + + const senders = (response.records ?? []).filter((record) => + (record.features ?? []).includes('SmsSender'), + ); + + if (senders.length === 0) { + return { + disabled: true, + // Distinguishes "nothing is SMS-enabled" from "the lookup failed", which look identical in an + // empty dropdown and send the reader to the wrong place. + placeholder: + 'No SMS-enabled numbers on this extension. Enable SMS on a number in the RingCentral admin portal.', + options: [], + }; + } + + return { + disabled: false, + options: senders.map((record) => ({ + label: record.usageType ? `${record.phoneNumber} (${record.usageType})` : record.phoneNumber, + value: record.phoneNumber, + })), + }; + }, +}); + +/** + * Team Messaging conversations, so posting does not require pasting a raw chat id. + */ +export const chatDropdown = Property.Dropdown({ + auth: ringcentralAuth, + displayName: 'Chat', + description: 'The direct message, group or team to post to.', + required: true, + refreshers: [], + options: async ({ auth }) => { + if (!auth) { + return notConnected(); + } + + const chats = await listAllChats(auth as OAuth2PropertyValue); + + return { + disabled: false, + options: chats.map((chat) => ({ + label: chat.name ?? `${chat.type ?? 'Chat'} (${chat.id})`, + value: chat.id, + })), + }; + }, +}); + +function notConnected() { + return { + disabled: true, + placeholder: 'Connect your account first', + options: [], + }; +} + +/** + * Follows the page tokens rather than reading the first page only. + * + * A single request caps at 250, and an account past that would silently be missing exactly the chats + * a user could not then select, with no hint that the list was truncated. The page cap stops a + * pathological account from hanging the dropdown; it is high enough that reaching it means the list + * was never going to be usable as a dropdown anyway. + */ +async function listAllChats(auth: OAuth2PropertyValue): Promise { + const MAX_PAGES = 10; + const collected: ChatRecord[] = []; + let pageToken: string | undefined = undefined; + + for (let page = 0; page < MAX_PAGES; page++) { + const queryParams: Record = { recordCount: '250' }; + if (pageToken) queryParams['pageToken'] = pageToken; + + const response: TokenPagedRecords = + await ringcentralCommon.sendRequest>({ + auth, + method: HttpMethod.GET, + resourcePath: '/team-messaging/v1/chats', + queryParams, + }); + + collected.push(...(response.records ?? [])); + + pageToken = response.navigation?.nextPageToken; + if (!pageToken) break; + } + + return collected; +} + +type PhoneNumberRecord = { + phoneNumber: string; + usageType?: string; + features?: string[]; +}; + +type ChatRecord = { + id: string; + name?: string; + type?: string; +}; + +type PagedRecords = { records?: T[] }; + +type TokenPagedRecords = { + records?: T[]; + navigation?: { nextPageToken?: string }; +}; diff --git a/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.test.ts b/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.test.ts new file mode 100644 index 000000000000..7767bdf824a3 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.test.ts @@ -0,0 +1,137 @@ +import { DEDUPE_KEY_PROPERTY } from '@activepieces/pieces-framework'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createSubscriptionTrigger } from './subscription-trigger'; +import { memStore, oauth, stubHttp } from './test-support/http-stub'; + +afterEach(() => vi.restoreAllMocks()); + +const STORE_KEY = 'ringcentral_test_events_subscription_id'; + +const trigger = createSubscriptionTrigger<{ id?: string | number; kind?: string }>({ + name: 'test_events', + displayName: 'Test Events', + description: 'Fixture trigger for the factory.', + eventFilters: ['/restapi/v1.0/test/events'], + accept: (body) => body.kind !== 'Rejected', + sampleData: { id: 1 }, +}); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const ctx = (over: Record = {}): any => ({ + auth: oauth(), + store: memStore(), + webhookUrl: 'https://example.com/webhook/abc', + payload: { headers: {}, body: {} }, + ...over, +}); + +describe('handshake', () => { + it('echoes the Validation-Token header back', async () => { + const res = await trigger.onHandshake( + ctx({ payload: { headers: { 'validation-token': 'tok-1' }, body: {} } }), + ); + expect(res).toEqual({ status: 200, headers: { 'Validation-Token': 'tok-1' } }); + }); + + it('rejects a handshake without the header', async () => { + const res = await trigger.onHandshake(ctx()); + expect(res.status).toBe(400); + }); +}); + +describe('lifecycle', () => { + it('creates the subscription on enable and remembers its id', async () => { + const stub = stubHttp(); + stub.route('/subscription', { id: 'sub-42' }); + const store = memStore(); + + await trigger.onEnable(ctx({ store })); + + expect(stub.find('/subscription')?.body).toMatchObject({ + eventFilters: ['/restapi/v1.0/test/events'], + deliveryMode: { address: 'https://example.com/webhook/abc' }, + }); + expect(await store.get(STORE_KEY)).toBe('sub-42'); + }); + + it('deletes the subscription and the stored id on disable', async () => { + const stub = stubHttp(); + stub.route('/subscription/sub-42', {}); + const store = memStore(); + await store.put(STORE_KEY, 'sub-42'); + + await trigger.onDisable(ctx({ store })); + + expect(stub.find('/subscription/sub-42')?.method).toBe('DELETE'); + expect(await store.get(STORE_KEY)).toBeNull(); + }); + + it('still clears the stored id when RingCentral already killed the subscription', async () => { + const stub = stubHttp(); + stub.route('/subscription/sub-42', () => new Error('gone')); + const store = memStore(); + await store.put(STORE_KEY, 'sub-42'); + + await expect(trigger.onDisable(ctx({ store }))).resolves.not.toThrow(); + expect(await store.get(STORE_KEY)).toBeNull(); + }); +}); + +describe('run', () => { + const enabled = async () => { + const store = memStore(); + await store.put(STORE_KEY, 'sub-42'); + return store; + }; + + const delivery = (body: unknown, subscriptionId = 'sub-42', uuid?: string) => ({ + payload: { headers: {}, body: { subscriptionId, uuid, body } }, + }); + + it('drops a delivery whose subscription id is not ours', async () => { + const store = await enabled(); + // RingCentral does not sign deliveries, so the minted id is the only authenticity check. + const out = await trigger.run(ctx({ store, ...delivery({ id: 7 }, 'sub-FORGED') })); + expect(out).toEqual([]); + }); + + it('drops everything when no subscription id is stored at all', async () => { + const out = await trigger.run(ctx({ ...delivery({ id: 7 }) })); + expect(out).toEqual([]); + }); + + it('drops a delivery the accept predicate rejects', async () => { + const store = await enabled(); + const out = await trigger.run(ctx({ store, ...delivery({ id: 7, kind: 'Rejected' }) })); + expect(out).toEqual([]); + }); + + it('keys accepted events by their id for platform dedupe', async () => { + const store = await enabled(); + const out = await trigger.run(ctx({ store, ...delivery({ id: 7, kind: 'Fine' }) })); + expect(out).toEqual([{ id: 7, kind: 'Fine', [DEDUPE_KEY_PROPERTY]: '7' }]); + }); + + it('falls back to the delivery uuid when the body has no id', async () => { + const store = await enabled(); + const out = await trigger.run(ctx({ store, ...delivery({ kind: 'Fine' }, 'sub-42', 'u-9') })); + expect(out).toEqual([{ kind: 'Fine', [DEDUPE_KEY_PROPERTY]: 'u-9' }]); + }); + + it('emits an id-less event un-keyed rather than keying every one to the same constant', async () => { + const store = await enabled(); + const out = await trigger.run(ctx({ store, ...delivery({ kind: 'Fine' }) })); + // A constant key ('') would make the platform swallow every later id-less event as a duplicate. + expect(out).toEqual([{ kind: 'Fine' }]); + expect(Object.prototype.hasOwnProperty.call(out[0], DEDUPE_KEY_PROPERTY)).toBe(false); + }); + + it('drops a delivery with no body', async () => { + const store = await enabled(); + const out = await trigger.run( + ctx({ store, payload: { headers: {}, body: { subscriptionId: 'sub-42' } } }), + ); + expect(out).toEqual([]); + }); +}); diff --git a/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.ts b/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.ts new file mode 100644 index 000000000000..5477fe8f2ecf --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/subscription-trigger.ts @@ -0,0 +1,104 @@ +import { + createTrigger, + DEDUPE_KEY_PROPERTY, + TriggerStrategy, + WebhookHandshakeStrategy, +} from '@activepieces/pieces-framework'; + +import { ringcentralAuth } from './auth'; +import { ringcentralCommon, RingCentralWebhookEvent } from './client'; + +/** + * All three triggers are the same machine: mint a WebHook subscription on enable, answer the + * Validation-Token handshake, tear the subscription down on disable, and filter deliveries. Only + * the event filters, the accept predicate and the sample payload differ, so those are the inputs. + */ +export function createSubscriptionTrigger< + T extends { id?: string | number }, +>({ + name, + displayName, + description, + eventFilters, + accept, + sampleData, +}: { + name: string; + displayName: string; + description: string; + eventFilters: string[]; + /** Keeps a delivery only when it is the event this trigger is about; omit to keep everything. */ + accept?: (body: T) => boolean; + sampleData: Record; +}) { + const subscriptionIdKey = `ringcentral_${name}_subscription_id`; + + return createTrigger({ + auth: ringcentralAuth, + name, + displayName, + description, + type: TriggerStrategy.WEBHOOK, + props: {}, + handshakeConfiguration: { + strategy: WebhookHandshakeStrategy.HEADER_PRESENT, + paramName: 'validation-token', + }, + async onHandshake(context) { + const validationToken = context.payload.headers['validation-token']; + if (!validationToken) { + return { status: 400, body: { message: 'Missing Validation-Token header.' } }; + } + // RingCentral proves the endpoint is ours by demanding its token echoed back in a header. + return { + status: 200, + headers: { 'Validation-Token': validationToken }, + }; + }, + async onEnable(context) { + const subscriptionId = await ringcentralCommon.createSubscription({ + auth: context.auth, + webhookUrl: context.webhookUrl, + eventFilters, + }); + await context.store.put(subscriptionIdKey, subscriptionId); + }, + async onDisable(context) { + const subscriptionId = await context.store.get(subscriptionIdKey); + if (subscriptionId) { + try { + await ringcentralCommon.deleteSubscription({ auth: context.auth, subscriptionId }); + } catch { + // The subscription may already be gone (expired, blacklisted after failed deliveries, or + // revoked on RingCentral's side); disabling the flow must not fail over cleanup. + } + } + await context.store.delete(subscriptionIdKey); + }, + async run(context) { + const event = (context.payload.body ?? {}) as RingCentralWebhookEvent; + + // RingCentral does not sign deliveries; the only secret a genuine one carries is the + // subscription id minted at onEnable, which never leaves the server. A POST to the webhook + // URL without that exact id is not from our subscription, so it is dropped, not trusted. + const expectedSubscriptionId = await context.store.get(subscriptionIdKey); + if (!expectedSubscriptionId || event.subscriptionId !== expectedSubscriptionId) { + return []; + } + + const body = event.body; + if (!body || (accept && !accept(body))) { + return []; + } + + // No id at all means no safe dedupe key: emitting a constant ('') would silently swallow + // every later id-less event as a duplicate, so such an event goes through un-keyed instead. + const dedupeKey = body.id ?? event.uuid; + if (dedupeKey === undefined || dedupeKey === null || dedupeKey === '') { + return [body]; + } + return [{ ...body, [DEDUPE_KEY_PROPERTY]: String(dedupeKey) }]; + }, + sampleData, + }); +} diff --git a/packages/pieces/community/ringcentral/src/lib/common/test-support/http-stub.ts b/packages/pieces/community/ringcentral/src/lib/common/test-support/http-stub.ts new file mode 100644 index 000000000000..9631cb4bc9e0 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/common/test-support/http-stub.ts @@ -0,0 +1,99 @@ +import { HttpError, httpClient } from '@activepieces/pieces-common'; +import { vi } from 'vitest'; + +export type Recorded = { + method: string; + url: string; + body?: Record; + queryParams?: Record; + timeout?: number; + retries?: number; + responseType?: string; + authentication?: { type: string; token?: string }; +}; + +export type Handler = Record | ((req: { url: string }) => unknown); + +/** Builds a real HttpError so `err instanceof HttpError` and its getters behave live. */ +export function httpError(status: number, responseBody: unknown): HttpError { + return new HttpError({}, { status, responseBody }); +} + +/** + * Replaces `httpClient.sendRequest` for the duration of a test file. Routes are matched by + * substring against the URL; an unrouted call fails loudly instead of returning a plausible + * empty object. + */ +export function stubHttp() { + const calls: Recorded[] = []; + const routes: Array<{ fragment: string; handler: Handler }> = []; + + vi.spyOn(httpClient, 'sendRequest').mockImplementation((async (req: Recorded) => { + calls.push({ + method: req.method, + url: req.url, + body: req.body, + queryParams: req.queryParams, + timeout: req.timeout, + retries: req.retries, + responseType: req.responseType, + authentication: req.authentication, + }); + + const route = routes.find((r) => req.url.includes(r.fragment)); + if (!route) throw new Error(`no stub route for ${req.url}`); + + const result = + typeof route.handler === 'function' ? route.handler({ url: req.url }) : route.handler; + if (result instanceof Error) throw result; + return { status: 200, headers: {}, body: result }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any); + + return { + calls, + route(fragment: string, handler: Handler) { + const existing = routes.findIndex((r) => r.fragment === fragment); + if (existing !== -1) routes.splice(existing, 1); + routes.push({ fragment, handler }); + }, + find: (fragment: string) => calls.find((c) => c.url.includes(fragment)), + }; +} + +/** In-memory stand-in for `context.store`. */ +export function memStore() { + const m = new Map(); + return { + map: m, + get: async (k: string) => (m.has(k) ? (m.get(k) as T) : null), + put: async (k: string, v: T) => { + m.set(k, v); + return v; + }, + delete: async (k: string) => { + m.delete(k); + }, + }; +} + +/** In-memory stand-in for `context.files`, which the attachment download writes through. */ +export function memFiles() { + const written: Array<{ fileName: string; data: Buffer }> = []; + return { + written, + write: async ({ fileName, data }: { fileName: string; data: Buffer }) => { + written.push({ fileName, data }); + return `mock://files/${fileName}`; + }, + }; +} + +/** An OAuth2 connection value as the platform hands it to a piece. */ +export function oauth(overrides: Record = {}) { + return { + access_token: 'RC_TOKEN', + props: { environment: 'platform.devtest.ringcentral.com' }, + ...overrides, + }; +} diff --git a/packages/pieces/community/ringcentral/src/lib/triggers/new-inbound-sms.ts b/packages/pieces/community/ringcentral/src/lib/triggers/new-inbound-sms.ts new file mode 100644 index 000000000000..10d86144bb25 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/triggers/new-inbound-sms.ts @@ -0,0 +1,42 @@ +import { createSubscriptionTrigger } from '../common/subscription-trigger'; + +export const newInboundSms = createSubscriptionTrigger<{ + id?: string | number; + direction?: string; +}>({ + name: 'new_inbound_sms', + displayName: 'New Inbound SMS or MMS', + description: + 'Triggers when a new inbound text message is received, with or without media attached.', + // ONE filter, and `type=SMS` is correct for picture messages too. Do not add `type=MMS`: there is + // no MMS message type. RingCentral delivers an inbound MMS through this same filter with + // `type: 'SMS'` and an `MmsAttachment` part, and an unrecognised type value can fail the whole + // createSubscription call, which would break the trigger rather than widen it. + // https://developers.ringcentral.com/guide/messaging/sms/receiving-sms-mms + eventFilters: ['/restapi/v1.0/account/~/extension/~/message-store/instant?type=SMS'], + // The instant message-store filter also fires for what this extension sends. + accept: (message) => message.direction === 'Inbound', + sampleData: { + uuid: '3c9d3d10-1f1a-4f0e-9b0e-2f2a9a1a4b6c', + id: 1234567890, + to: [{ phoneNumber: '+14155550100', name: 'RingCentral User' }], + from: { phoneNumber: '+14155550123' }, + type: 'SMS', + creationTime: '2024-01-15T18:30:00.000Z', + readStatus: 'Unread', + priority: 'Normal', + // A text-only SMS carries just the Text part. An inbound MMS looks identical except for an + // extra MmsAttachment part, which is what Download Message Attachment fetches. + attachments: [ + { id: 111, type: 'Text', contentType: 'text/plain' }, + { id: 222, type: 'MmsAttachment', contentType: 'image/jpeg' }, + ], + direction: 'Inbound', + availability: 'Alive', + subject: 'Hello from a customer!', + messageStatus: 'Received', + conversationId: 9876543210, + conversation: { id: '9876543210' }, + lastModifiedTime: '2024-01-15T18:30:00.000Z', + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/triggers/new-team-message.ts b/packages/pieces/community/ringcentral/src/lib/triggers/new-team-message.ts new file mode 100644 index 000000000000..eb5b6e432af7 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/triggers/new-team-message.ts @@ -0,0 +1,24 @@ +import { createSubscriptionTrigger } from '../common/subscription-trigger'; + +export const newTeamMessage = createSubscriptionTrigger<{ + id?: string | number; + eventType?: string; +}>({ + name: 'new_team_message', + displayName: 'New Team Messaging Post', + description: 'Triggers when a new post is added in RingCentral Team Messaging.', + eventFilters: ['/restapi/v1.0/glip/posts'], + // The posts filter also fires for edits and removals; only additions are this trigger. + accept: (post) => post.eventType === 'PostAdded', + sampleData: { + id: '5544332211', + groupId: '112233445566', + type: 'TextMessage', + text: 'Hey team, the deploy is done!', + creatorId: '778899', + addedPersonIds: ['778899'], + creationTime: '2024-01-15T20:10:00.000Z', + lastModifiedTime: '2024-01-15T20:10:00.000Z', + eventType: 'PostAdded', + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/triggers/new-voicemail.ts b/packages/pieces/community/ringcentral/src/lib/triggers/new-voicemail.ts new file mode 100644 index 000000000000..2a92af744143 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/triggers/new-voicemail.ts @@ -0,0 +1,31 @@ +import { createSubscriptionTrigger } from '../common/subscription-trigger'; + +export const newVoicemail = createSubscriptionTrigger<{ + id?: string | number; +}>({ + name: 'new_voicemail', + displayName: 'New Voicemail', + description: 'Triggers when a new voicemail message is received.', + // The dedicated voicemail filter, NOT message-store/instant. The instant filter is documented for + // inbound SMS only, so `?type=VoiceMail` either fails validation on enable or enables and never + // delivers. https://developers.ringcentral.com/guide/notifications/event-filters/voicemail-message + eventFilters: ['/restapi/v1.0/account/~/extension/~/voicemail'], + sampleData: { + uuid: '5a1b2c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d', + id: 2233445566, + to: [{ phoneNumber: '+14155550100', name: 'RingCentral User' }], + from: { phoneNumber: '+14155550123', name: 'Jane Caller' }, + type: 'VoiceMail', + creationTime: '2024-01-15T19:05:00.000Z', + readStatus: 'Unread', + priority: 'Normal', + attachments: [ + { id: 222, type: 'AudioRecording', contentType: 'audio/mpeg', vmDuration: 12 }, + ], + direction: 'Inbound', + availability: 'Alive', + messageStatus: 'Received', + vmTranscriptionStatus: 'NotAvailable', + lastModifiedTime: '2024-01-15T19:05:00.000Z', + }, +}); diff --git a/packages/pieces/community/ringcentral/src/lib/triggers/triggers.test.ts b/packages/pieces/community/ringcentral/src/lib/triggers/triggers.test.ts new file mode 100644 index 000000000000..af2d1cfc0aa2 --- /dev/null +++ b/packages/pieces/community/ringcentral/src/lib/triggers/triggers.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { memStore, oauth, stubHttp } from '../common/test-support/http-stub'; +import { newInboundSms } from './new-inbound-sms'; +import { newTeamMessage } from './new-team-message'; +import { newVoicemail } from './new-voicemail'; + +afterEach(() => vi.restoreAllMocks()); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const ctx = (): any => ({ + auth: oauth(), + store: memStore(), + webhookUrl: 'https://example.com/webhook/abc', + payload: { headers: {}, body: {} }, +}); + +/** The event filters a trigger actually asks RingCentral to subscribe to. */ +async function filtersFor(trigger: { onEnable: (c: unknown) => Promise }) { + const stub = stubHttp(); + stub.route('/subscription', { id: 'sub-1' }); + await trigger.onEnable(ctx()); + return (stub.find('/subscription')?.body as { eventFilters?: string[] })?.eventFilters; +} + +describe('subscribed event filters', () => { + it('subscribes the inbound text trigger to type=SMS only, which already covers MMS', async () => { + // Guards against "widening" this to type=MMS. There is no MMS message type: RingCentral delivers + // an inbound picture message through this same filter with type: 'SMS' plus an MmsAttachment + // part, and an unrecognised type can fail createSubscription outright, breaking the trigger + // rather than widening it. + // https://developers.ringcentral.com/guide/messaging/sms/receiving-sms-mms + expect(await filtersFor(newInboundSms)).toEqual([ + '/restapi/v1.0/account/~/extension/~/message-store/instant?type=SMS', + ]); + }); + + it('subscribes voicemail to its own dedicated filter, not message-store/instant', async () => { + // message-store/instant is documented for inbound SMS only, so ?type=VoiceMail either fails + // validation on enable or enables and never delivers. + // https://developers.ringcentral.com/guide/notifications/event-filters/voicemail-message + expect(await filtersFor(newVoicemail)).toEqual([ + '/restapi/v1.0/account/~/extension/~/voicemail', + ]); + }); + + it('subscribes team messaging to the glip posts feed', async () => { + expect(await filtersFor(newTeamMessage)).toEqual(['/restapi/v1.0/glip/posts']); + }); +}); + +describe('inbound text filtering', () => { + it('keeps an inbound picture message, which arrives as type SMS with an MmsAttachment', async () => { + const store = memStore(); + await store.put('ringcentral_new_inbound_sms_subscription_id', 'sub-1'); + const result = await newInboundSms.run({ + store, + payload: { + headers: {}, + body: { + subscriptionId: 'sub-1', + body: { + id: 5, + direction: 'Inbound', + type: 'SMS', + attachments: [{ id: 222, type: 'MmsAttachment', contentType: 'image/jpeg' }], + }, + }, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); + expect(result).toHaveLength(1); + }); + + it('drops what this extension sent itself', async () => { + const store = memStore(); + await store.put('ringcentral_new_inbound_sms_subscription_id', 'sub-1'); + const result = await newInboundSms.run({ + store, + payload: { + headers: {}, + body: { subscriptionId: 'sub-1', body: { id: 6, direction: 'Outbound', type: 'SMS' } }, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); + expect(result).toEqual([]); + }); +}); diff --git a/packages/pieces/community/ringcentral/tsconfig.json b/packages/pieces/community/ringcentral/tsconfig.json new file mode 100644 index 000000000000..7bef1d125014 --- /dev/null +++ b/packages/pieces/community/ringcentral/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ], + "compilerOptions": { + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + } +} diff --git a/packages/pieces/community/ringcentral/tsconfig.lib.json b/packages/pieces/community/ringcentral/tsconfig.lib.json new file mode 100644 index 000000000000..8618ce322a7f --- /dev/null +++ b/packages/pieces/community/ringcentral/tsconfig.lib.json @@ -0,0 +1,20 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "rootDir": ".", + "baseUrl": ".", + "paths": {}, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "types": ["node"] + }, + "exclude": [ + "jest.config.ts", + "src/**/*.spec.ts", + "src/**/*.test.ts", + "src/**/test-support/**" + ], + "include": ["src/**/*.ts"] +} diff --git a/packages/pieces/community/ringcentral/vitest.config.ts b/packages/pieces/community/ringcentral/vitest.config.ts new file mode 100644 index 000000000000..f520fc141133 --- /dev/null +++ b/packages/pieces/community/ringcentral/vitest.config.ts @@ -0,0 +1,18 @@ +import path from 'path' +import { defineConfig } from 'vitest/config' + +const repoRoot = path.resolve(__dirname, '../../../..') + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + }, + resolve: { + alias: { + '@activepieces/shared': path.resolve(repoRoot, 'packages/core/shared/src/index.ts'), + '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'), + }, + }, +}) From 1c6efa0389f9d09e818b5b75072aa129ebae42a2 Mon Sep 17 00:00:00 2001 From: Mo AbuAboud Date: Tue, 18 Aug 2026 17:51:16 +0300 Subject: [PATCH 3/6] fix(engine): run pieces in a fresh child process (#14889) --- ...orts-a-piece-a-fresh-child-process-does.md | 31 ++ brain/knowledge/execution-runtime/index.md | 2 + bun.lock | 5 +- packages/core/execution/package.json | 2 +- .../src/lib/engine/execution-errors.ts | 10 + packages/server/engine/esbuild.config.mjs | 24 +- packages/server/engine/package.json | 3 +- .../engine/src/lib/core/piece/piece-auth.ts | 101 +++++ .../engine/src/lib/core/piece/piece-child.ts | 107 +++++ .../lib/core/piece/piece-context-builder.ts | 326 ++++++++++++++++ .../engine/src/lib/core/piece/piece-path.ts | 142 +++++++ .../src/lib/core/piece/piece-protocol.ts | 160 ++++++++ .../engine/src/lib/core/piece/piece-runner.ts | 153 ++++++++ .../src/lib/core/piece/trigger-runner.ts | 206 ++++++++++ .../engine/src/lib/handler/flow-executor.ts | 4 +- .../engine/src/lib/handler/piece-executor.ts | 260 +++---------- .../lib/helper/flow-run-progress-reporter.ts | 16 +- .../engine/src/lib/helper/piece-helper.ts | 366 ------------------ .../engine/src/lib/helper/piece-loader.ts | 282 -------------- .../engine/src/lib/helper/trigger-helper.ts | 301 -------------- .../lib/operations/auth-refresh.operation.ts | 29 +- .../operations/auth-validation.operation.ts | 33 +- .../src/lib/operations/flow.operation.ts | 4 +- .../operations/piece-metadata.operation.ts | 27 +- .../src/lib/operations/property.operation.ts | 138 ++++++- ...resolve-connection-identifier.operation.ts | 12 +- .../lib/operations/trigger-hook.operation.ts | 4 +- packages/server/engine/src/piece-child.ts | 3 + .../test/core/piece/piece-memory.test.ts | 26 ++ .../test/core/piece/piece-protocol.test.ts | 48 +++ .../engine/test/handler/flow-log-size.test.ts | 4 +- .../handler/flow-waitpoint-response.test.ts | 22 +- .../test/handler/flow-with-delay.test.ts | 39 +- .../test/handler/flow-with-pause.test.ts | 29 +- .../helper/flow-run-progress-reporter.test.ts | 4 +- .../engine/test/helpers/engine-api-stub.ts | 65 ++++ .../flow-operation-invariants.test.ts | 61 ++- .../operations/trigger-hook-operation.test.ts | 4 +- packages/server/engine/vitest.config.ts | 34 +- .../src/lib/cache/engine/engine-installer.ts | 16 +- .../cache/engine-installer-identity.test.ts | 3 + 41 files changed, 1790 insertions(+), 1316 deletions(-) create mode 100644 brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md create mode 100644 packages/server/engine/src/lib/core/piece/piece-auth.ts create mode 100644 packages/server/engine/src/lib/core/piece/piece-child.ts create mode 100644 packages/server/engine/src/lib/core/piece/piece-context-builder.ts create mode 100644 packages/server/engine/src/lib/core/piece/piece-path.ts create mode 100644 packages/server/engine/src/lib/core/piece/piece-protocol.ts create mode 100644 packages/server/engine/src/lib/core/piece/piece-runner.ts create mode 100644 packages/server/engine/src/lib/core/piece/trigger-runner.ts delete mode 100644 packages/server/engine/src/lib/helper/piece-helper.ts delete mode 100644 packages/server/engine/src/lib/helper/piece-loader.ts delete mode 100644 packages/server/engine/src/lib/helper/trigger-helper.ts create mode 100644 packages/server/engine/src/piece-child.ts create mode 100644 packages/server/engine/test/core/piece/piece-memory.test.ts create mode 100644 packages/server/engine/test/core/piece/piece-protocol.test.ts create mode 100644 packages/server/engine/test/helpers/engine-api-stub.ts diff --git a/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md b/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md new file mode 100644 index 000000000000..f79ea7bba5ac --- /dev/null +++ b/brain/knowledge/decisions/000029-the-engine-never-imports-a-piece-a-fresh-child-process-does.md @@ -0,0 +1,31 @@ +--- +status: accepted +--- + +# The engine never imports a piece, a fresh child process does + +## Decision + +Nothing in the engine process may `import()` a piece package. A piece is loaded only inside a child process spawned per call and killed when that call returns (`packages/server/engine/src/lib/core/piece/piece-child.ts`, shipped as its own esbuild entry `piece-child.js`). The parent talks to it with exactly two requests — `describe` (piece metadata as JSON plus the list of paths that are functions) and `call` (`['actions', 'send_http', 'run']` and its arguments) — over `piece-runner.ts`. + +## Context + +The engine is long-lived and served many operations, each `import()`ing pieces into the same process. Resident piece modules (and their duplicated `@activepieces/shared` copies) never came back — measured as hundreds of MB of a single engine heap. Loading a piece to read its metadata, or just to discover an auth `validate` hook does not exist, cost the same permanent memory as running it. Measured after the change with `smoke-test/verify-memory.sh` (webhook → data-mapper → return-response, 2000 runs): the engine ends **28 MB below** its warm baseline, i.e. V8 gives the heap back because nothing from the pieces stays resident. + +## Why + +Process exit is the only reliable way to free a required module graph; a cache or a `delete require.cache` does not free native handles or the transitive graph. Everything the engine needs about a piece is data (props, auth, trigger type, `contextInfo`), so it can cross a process boundary — only *behaviour* has to run where the piece is loaded. Rejected: keeping metadata loading in-process and isolating only `run` (metadata loading is what most operations do, so the leak would remain), and a persistent piece process per version (it re-creates the leak with extra lifecycle). + +The child is a real bundled engine entry, not an inline `--eval` script, because file materialization must live with the engine's own file processor: `ApStreamingFile.body` is a `Readable` and cannot be structured-cloned. + +## Consequences + +- **The piece's context cannot be proxied back to the engine — the child has to build it.** Two hard constraints kill any marker/RPC bridge, and both fail silently: (1) parts of the context are **synchronous by contract** — `CreateWaitpointResult.buildResumeUrl` returns a `string` and pieces call it without `await`, which an RPC can only answer with a Promise; (2) pieces **mutate objects after handing them to a hook** — `return-response-and-wait-for-next-webhook` passes `response` to `createWaitpoint` and only then writes the resume URL into its `headers`, and in-process the engine sees that through the shared reference. A snapshot does not. Since almost every context function is just HTTP over scalars (`apiUrl`, `engineToken`, `projectId`, `flowId`) the child builds them itself; only the collectors the engine reads afterwards (`hookResponse` tags/stop/respond/paused/responseToSend, trigger `listeners` and `scheduleOptions`) travel back, as plain data on the result. +- `describe` costs one extra spawn per piece per engine process (memoized by `name@version`), so a 10-step flow on one piece is 11 spawns, not 20. +- **A piece call costs ~79 ms** (spawn + import the piece + build the context + run + IPC), measured on a warm cache against the built child bundle. The benchmark flow's three calls show up as `RUN=400ms` in `FlowRun.timeline` with `PROVISION=0ms, BOOT=0ms`. That is the price of never letting a piece into the engine heap; if the sync-webhook path ever needs it back, the upgrade is one child per *flow run* instead of per *call* — the process still dies at the end of the run, so nothing accumulates, and an N-step flow pays one spawn instead of N. +- `fileProcessor` returns a `__apFileSource` marker and the child calls `materializeFile`, so nothing is fetched until the piece actually runs — a validation failure now opens zero connections. The cost: an unreachable file URL fails *in the child when the step runs* rather than in the parent's prop validation (same message, later stage), because you cannot check a remote file without fetching it. +- Piece metadata reaches the engine JSON-round-tripped, so any *function* on a property (dropdown `options`, dynamic `props`) is addressable only by path, never callable in-process. +- A sandbox gets the engine by **file-by-file copy**, not by copying a directory: `engineInstaller` (`packages/server/sandbox`) copies each bundle into the cache dir that isolate mounts at `/root/common`. A new engine entry point must be added to that list or it is simply absent at runtime — the Docker image, which copies all of `dist/packages/engine`, looks perfectly fine and hides it. +- The child is a second esbuild entry, so anything that builds it (including `vitest.config.ts`, which builds it for tests) must reuse the same `alias` map as `esbuild.config.mjs` — miss it and tests bundle `@activepieces/*` from `dist` while production bundles from `src`, so a green suite proves nothing about the shipped child. +- **The child inherits the engine's node flags, and its OOM is detected rather than prevented.** `engineNodeArgs` sets `--max-old-space-size` as the *only* bound on engine memory (isolate passes no `--mem`/`--cg-mem`), so spawning the child without it would leave it on V8's default heap — spawn it with `[...process.execArgv, entry]`. Engine + child can then together reach `AP_SANDBOX_MEMORY_LIMIT`, and that is accepted: the worker is the sandbox, so it dies and restarts. What must not happen is the failure being anonymous — `piece-runner` classifies the child's exit the way `sandbox.ts` classifies the engine's (V8 heap message, exit 134, SIGABRT, SIGKILL) and raises a user-level `PieceMemoryLimitError`. A budget split between the two processes was tried and reverted: it bought little, and a floor on each share silently overshot the limit on small configurations. +- The child inherits no in-process guards the parent installs (e.g. the SSRF monkeypatches in `network/ssrf-guard.ts`). Whatever must apply to piece code has to be installed in `src/piece-child.ts`. diff --git a/brain/knowledge/execution-runtime/index.md b/brain/knowledge/execution-runtime/index.md index 899ef70ff7c8..d3456d9df1a2 100644 --- a/brain/knowledge/execution-runtime/index.md +++ b/brain/knowledge/execution-runtime/index.md @@ -48,6 +48,8 @@ The four calls a run emits to the app during execution: `updateRunProgress`, `up - **The S3 piece-tarball cache shadows the CDN, so changing *what* gets cached means bumping `S3_PIECES_PREFIX`, not purging it.** `resolve()` (`piece-bundle.ts`) checks S3 before the CDN, so whatever `BUNDLE_PIECE` wrote wins for every later request. Until Aug 2026 that job cached the **npm** tarball, which for versions published before piece repackaging still declares its build-time deps — measured cost: 12 resident `@activepieces/shared` versions holding 388 MB of a 554 MB engine heap on cloud. The job now prefers the CDN artifact, but fixing the writer does not fix the objects already written, and *purging* them cannot work: a rolling deploy leaves old app instances writing npm tarballs back into the prefix for the rest of the rollout, and the purge has no way to know when the last one is gone. So the prefix is versioned (`pieces/` → `pieces/v2/`) — old code can only write the old prefix, so the new one is reachable only by a CDN-preferring writer. Same reflex as `LATEST_CACHE_VERSION` on the worker: when the meaning of a cached value changes, move the key; the abandoned prefix is dead storage to be swept later, never a correctness dependency. - **`extractConnectionIds` misses agent-tool connections.** It only reads step/trigger `settings.input.auth`, never `agentTools[].pieceMetadata.predefinedInput.auth`, so `flowVersion.connectionIds` under-reports and "which flows use this connection" lies. - **A code-sandbox `functions` entry must be a standalone declaration, never an object-method shorthand.** The v8 isolate re-injects each entry as source via `const ${key} = ${value.toString()}` (`v8-isolate-code-sandbox.ts`). A standalone `function flattenNestedKeys(...) {...}` (as exported from `script-evaluator.ts`) stringifies to a valid RHS and keeps recursion working by its inner name; an inline object-method shorthand stringifies to `flattenNestedKeys(...) {...}`, a syntax error as a `const` RHS. Keep it a standalone `function` export, never a method. For the same reason do **not** relocate a sandbox-injected function behind a separately-built package boundary (e.g. `@activepieces/core-utils`): its serialized `.toString()` would then depend on that package's build/minify config staying isolate-friendly. The trap: `no-op-code-sandbox.ts` passes the function by reference and tolerates either form, so a test run that skips the isolated-vm suite ships the bug green. Related: the `functions` **key** is also the global name users type in flow inputs (`{{flattenNestedKeys(...)}}`), so it is a public contract string, not an implementation detail. Keep it a hardcoded literal (matched by `FLATTEN_NESTED_KEYS_PATTERN` in `props-resolver.ts`); never derive it from the function's `.name`, which mangles under minification and would wrongly couple the token to the JS identifier. +- **The piece context is lazier and more mutable than it reads.** Three traps when assembling it anywhere new (they all surfaced when context assembly moved into the piece child process, `core/piece/piece-context-builder.ts`): `project.externalId` is a **function the piece calls**, not a value — resolving it while building the context fires a `/v1/worker/project` request on *every step*; the backward-compatibility wrapper (`backwardCompatabilityContextUtils.makeActionContextBackwardCompatible`) must wrap the finished context or pieces on older context versions die with `ctx.run.pause is not a function`; and the legacy pause shim calls `createWaitpoint()` **without awaiting it**, so whoever owns the context has to drain in-flight hook work before the process ends or the waitpoint POST never lands and the run hangs until timeout. +- **An error loses its friendly HTTP details the moment it crosses a process boundary.** `formatPieceError` (`friendly-piece-error.ts`) reads `error.response.{status,body}`, `error.status`, and falls back to `error.constructor.name` for `errorName` — but on `HttpError` (`pieces-common`) `response` is a **prototype getter** and `name` is plain `'Error'`. Structured clone, `{...e}`, and `JSON.stringify` all copy own enumerable props only, so a child-process runner that ships an error back verbatim silently drops `status`, `apiMessage`, and the error name, and the step renders as an opaque JSON blob. Serialize errors explicitly: read the getter keys by name (`response`, `request`, `status`, `headers`, `body`, `error`) plus own props, and carry `constructor.name` as `name`. Same trap applies to the run **result**: it must be JSON round-tripped, or an unresolved promise/function anywhere in the returned object throws `could not be cloned` from `process.send` and fails the step. - **A props-resolver script session is per-`resolve()`, never shared or hoisted.** `getPropsResolver(...).resolve(...)` builds a fresh `PropsResolver` per call, creates the script session via `scriptEvaluator.initSession()`, and disposes it in `resolve`'s `finally`, so an instance is single-use. Freshness is load-bearing: `setGlobal` is no-overwrite (`v8-isolate-code-sandbox.ts`) and injects each referenced step view once per resolve, so a session reused across resolves serves **stale step views** as flow state advances, and a reused instance would run on an already-disposed session. When refactoring props-resolver, capture `getStepView` and `scriptSession` inside `resolve` (they depend on the per-call `executionState`), not at instance scope, and never behind a shared mutable variable. --- diff --git a/bun.lock b/bun.lock index e0f522eb4072..ff3ecf1f1d7c 100644 --- a/bun.lock +++ b/bun.lock @@ -117,7 +117,7 @@ }, "packages/core/execution": { "name": "@activepieces/core-execution", - "version": "0.13.0", + "version": "0.14.0", "dependencies": { "@activepieces/core-piece-types": "workspace:*", "@activepieces/core-utils": "workspace:*", @@ -162,7 +162,7 @@ }, "packages/core/shared": { "name": "@activepieces/shared", - "version": "0.137.0", + "version": "0.138.0", "dependencies": { "@activepieces/core-execution": "workspace:*", "@activepieces/core-formula": "workspace:*", @@ -10782,6 +10782,7 @@ }, "devDependencies": { "@types/node": "24.11.0", + "esbuild": "0.28.1", "form-data": "4.0.6", "vitest": "3.2.6", }, diff --git a/packages/core/execution/package.json b/packages/core/execution/package.json index c6257fd9a821..f070ef06aa7b 100644 --- a/packages/core/execution/package.json +++ b/packages/core/execution/package.json @@ -1,6 +1,6 @@ { "name": "@activepieces/core-execution", - "version": "0.13.0", + "version": "0.14.0", "type": "commonjs", "main": "./dist/src/index.js", "scripts": { diff --git a/packages/core/execution/src/lib/engine/execution-errors.ts b/packages/core/execution/src/lib/engine/execution-errors.ts index 49680320d560..88fb2f21f01d 100644 --- a/packages/core/execution/src/lib/engine/execution-errors.ts +++ b/packages/core/execution/src/lib/engine/execution-errors.ts @@ -84,6 +84,16 @@ export class PausedFlowTimeoutError extends ExecutionError { } } +export class PieceMemoryLimitError extends ExecutionError { + constructor(heapLimitMb: string | undefined, standardError?: string, cause?: unknown) { + super('PieceMemoryLimitError', JSON.stringify({ + message: 'The piece ran out of memory', + heapLimitMb, + standardError, + }), ExecutionErrorType.USER, cause) + } +} + export class FileSizeError extends ExecutionError { constructor(currentFileSize: number, maximumSupportSize: number, cause?: unknown) { super('FileSizeError', JSON.stringify({ diff --git a/packages/server/engine/esbuild.config.mjs b/packages/server/engine/esbuild.config.mjs index 83915618b411..ab3f5c079a5f 100644 --- a/packages/server/engine/esbuild.config.mjs +++ b/packages/server/engine/esbuild.config.mjs @@ -6,6 +6,7 @@ import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const outdir = path.resolve(__dirname, '../../../dist/packages/engine'); const proxyOutfile = path.join(outdir, 'main.js'); +const pieceChildOutfile = path.join(outdir, 'piece-child.js'); const watch = process.argv.includes('--watch'); @@ -54,9 +55,9 @@ function rebuildLogger(outfile) { }; } -function buildOptions({ outfile }) { +function buildOptions({ outfile, entry = 'src/main.ts' }) { return { - entryPoints: [path.resolve(__dirname, 'src/main.ts')], + entryPoints: [path.resolve(__dirname, entry)], bundle: true, platform: 'node', target: 'node20', @@ -80,14 +81,17 @@ function buildOptions({ outfile }) { }; } +const targets = [ + buildOptions({ outfile: proxyOutfile }), + buildOptions({ outfile: pieceChildOutfile, entry: 'src/piece-child.ts' }), +]; + if (watch) { - const ctx = await esbuild.context( - buildOptions({ outfile: proxyOutfile }) - ); - await ctx.rebuild(); - await ctx.watch(); + for (const target of targets) { + const ctx = await esbuild.context(target); + await ctx.rebuild(); + await ctx.watch(); + } } else { - await esbuild.build( - buildOptions({ outfile: proxyOutfile }) - ); + await Promise.all(targets.map((target) => esbuild.build(target))); } diff --git a/packages/server/engine/package.json b/packages/server/engine/package.json index 09b7e63b6653..86b539cddf5c 100644 --- a/packages/server/engine/package.json +++ b/packages/server/engine/package.json @@ -32,6 +32,7 @@ "devDependencies": { "@types/node": "24.11.0", "form-data": "4.0.6", - "vitest": "3.2.6" + "vitest": "3.2.6", + "esbuild": "0.28.1" } } diff --git a/packages/server/engine/src/lib/core/piece/piece-auth.ts b/packages/server/engine/src/lib/core/piece/piece-auth.ts new file mode 100644 index 000000000000..1761a28c7139 --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-auth.ts @@ -0,0 +1,101 @@ +import { isNil } from '@activepieces/core-utils' +import { getAuthPropertyForValue, PieceAuthProperty, PropertyType } from '@activepieces/pieces-framework' +import { AppConnectionType, AppConnectionValue, PiecePackage } from '@activepieces/shared' +import { EngineConstants } from '../../handler/context/engine-constants' +import { PieceDescription } from './piece-protocol' +import { PieceRef, pieceRunner } from './piece-runner' + +export const pieceAuth = { + callMethod: async ({ operation, authValueType, methodPath }: CallMethodParams): Promise => { + const piece: PieceRef = { + pieceName: operation.piece.pieceName, + pieceVersion: operation.piece.pieceVersion, + devPieces: EngineConstants.DEV_PIECES, + } + const description = await pieceRunner.describe(piece) + const selected = select({ description, authValueType }) + if (isNil(selected)) { + return { called: false } + } + const path = [...selected.path, ...methodPath] + if (!description.hasPath(path)) { + return { called: false, property: selected.property } + } + const argument = argumentFor({ property: selected.property, value: operation.auth }) + if (isNil(argument)) { + return { called: false, property: selected.property, mismatch: true } + } + const server = { + apiUrl: operation.internalApiUrl.endsWith('/') ? operation.internalApiUrl : `${operation.internalApiUrl}/`, + publicUrl: operation.publicApiUrl, + } + return { + called: true, + property: selected.property, + result: await pieceRunner.call({ piece, path, args: [{ auth: argument.argument, server }] }), + } + }, +} + +function select({ description, authValueType }: SelectParams): SelectedAuth | undefined { + const auth = description.metadata.auth + if (isNil(auth)) { + return undefined + } + const property = getAuthPropertyForValue({ authValueType, pieceAuth: auth }) + if (isNil(property)) { + return undefined + } + const index = Array.isArray(auth) ? auth.indexOf(property) : -1 + return { + property, + path: index === -1 ? ['auth'] : ['auth', String(index)], + } +} + +function argumentFor({ property, value }: ArgumentParams): { argument: unknown } | undefined { + switch (property.type) { + case PropertyType.OAUTH2: + return [AppConnectionType.OAUTH2, AppConnectionType.CLOUD_OAUTH2, AppConnectionType.PLATFORM_OAUTH2].includes(value.type) ? { argument: value } : undefined + case PropertyType.BASIC_AUTH: + return value.type === AppConnectionType.BASIC_AUTH ? { argument: value } : undefined + case PropertyType.SECRET_TEXT: + return value.type === AppConnectionType.SECRET_TEXT ? { argument: value.secret_text } : undefined + case PropertyType.CUSTOM_AUTH: + return value.type === AppConnectionType.CUSTOM_AUTH ? { argument: value.props } : undefined + case PropertyType.OIDC: + return value.type === AppConnectionType.OIDC ? { argument: value.props } : undefined + default: + return undefined + } +} + +type SelectParams = { + description: PieceDescription + authValueType: AppConnectionType +} + +type ArgumentParams = { + property: PieceAuthProperty + value: AppConnectionValue +} + +type SelectedAuth = { + property: PieceAuthProperty + path: string[] +} + +type CallMethodParams = { + operation: { + piece: PiecePackage + auth: AppConnectionValue + internalApiUrl: string + publicApiUrl: string + } + authValueType: AppConnectionType + methodPath: string[] +} + +export type AuthCallResult = + | { called: false, property?: PieceAuthProperty, mismatch?: boolean } + | { called: true, property: PieceAuthProperty, result: unknown } diff --git a/packages/server/engine/src/lib/core/piece/piece-child.ts b/packages/server/engine/src/lib/core/piece/piece-child.ts new file mode 100644 index 000000000000..9fe447e159fe --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-child.ts @@ -0,0 +1,107 @@ +import { isNil, isObject } from '@activepieces/core-utils' +import { Piece } from '@activepieces/pieces-framework' +import { extractPieceFromModule } from '@activepieces/shared' +import { buildContext } from './piece-context-builder' +import { ChildMessage, ParentMessage, pieceProtocol } from './piece-protocol' + +export const pieceChild = { + listen: (): void => { + process.on('message', (message: ParentMessage) => void handleParentMessage(message)) + process.on('disconnect', () => process.exit(0)) + process.on('unhandledRejection', (reason) => report({ type: 'done', success: false, error: pieceProtocol.serializeError(reason) })) + process.on('uncaughtException', (error) => report({ type: 'done', success: false, error: pieceProtocol.serializeError(error) })) + }, +} + +async function handleParentMessage(message: ParentMessage): Promise { + try { + const piece = await loadPiece(message) + if (message.type === 'describe') { + report({ type: 'done', success: true, result: describe(piece) }) + return + } + const built = isNil(message.context) ? undefined : await buildContext({ piece, request: message.context }) + const method = resolveMethod({ piece, path: message.path }) + const result = await method.call(...[...message.args, ...built?.args ?? []]) + await Promise.allSettled(built?.pending ?? []) + report({ + type: 'done', + success: true, + result: pieceProtocol.toTransferable(result), + hooks: built?.hooks, + }) + } + catch (error) { + report({ type: 'done', success: false, error: pieceProtocol.serializeError(error) }) + } +} + +async function loadPiece({ piecePath, pieceName, pieceVersion }: { piecePath: string, pieceName: string, pieceVersion: string }): Promise { + const pieceModule = await import(piecePath) + return extractPieceFromModule({ module: pieceModule, pieceName, pieceVersion }) +} + +function describe(piece: Piece): unknown { + return { + metadata: pieceProtocol.toTransferable(piece.metadata()), + functionPaths: collectFunctionPaths({ value: callableRoot(piece), path: [], depth: 0, seen: new Set() }), + } +} + +function callableRoot(piece: Piece): Record { + return { + actions: piece.actions(), + triggers: piece.triggers(), + auth: piece.auth, + events: piece.events, + } +} + +function resolveMethod({ piece, path }: { piece: Piece, path: string[] }): BoundMethod { + let owner: unknown = undefined + let current: unknown = callableRoot(piece) + for (const segment of path) { + if (!isObject(current)) { + throw new Error(`Path not found in piece: ${path.join('.')}`) + } + owner = current + current = Reflect.get(current, segment) + } + if (typeof current !== 'function') { + throw new Error(`Path is not callable in piece: ${path.join('.')}`) + } + const method = current + return { call: async (...args: unknown[]) => method.apply(owner, args) } +} + +function collectFunctionPaths({ value, path, depth, seen }: CollectParams): string[] { + if (depth > MAX_FUNCTION_PATH_DEPTH || !isObject(value) || seen.has(value)) { + return [] + } + return Object.entries(value).flatMap(([key, item]) => { + const itemPath = [...path, key] + return typeof item === 'function' ? [itemPath.join('.')] : collectFunctionPaths({ value: item, path: itemPath, depth: depth + 1, seen: new Set([...seen, value]) }) + }) +} + +function report(message: ChildMessage): void { + if (settled) { + return + } + settled = true + process.send?.(message, () => process.exit(0)) +} + +const MAX_FUNCTION_PATH_DEPTH = 6 +let settled = false + +type BoundMethod = { + call: (...args: unknown[]) => Promise +} + +type CollectParams = { + value: unknown + path: string[] + depth: number + seen: Set +} diff --git a/packages/server/engine/src/lib/core/piece/piece-context-builder.ts b/packages/server/engine/src/lib/core/piece/piece-context-builder.ts new file mode 100644 index 000000000000..69561c05fec9 --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-context-builder.ts @@ -0,0 +1,326 @@ +import { isNil, isObject } from '@activepieces/core-utils' +import { ActionContext, backwardCompatabilityContextUtils, CreateWaitpointHook, CreateWaitpointParams, CreateWaitpointResult, InputPropertyMap, Piece, PieceAuthProperty, PiecePropertyMap, SetScheduleRequest, StaticPropsValue, StopHookParams, TagsManager } from '@activepieces/pieces-framework' +import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, InvalidCronExpressionError, InvalidScheduleIntervalError, PausedFlowTimeoutError, ScheduleOptions, TriggerSourceScheduleType } from '@activepieces/shared' +import { isValidCron } from 'cron-validator' +import dayjs from 'dayjs' +import { retryFetch } from '../../api/retry-fetch' +import { flowRunProgressReporter } from '../../helper/flow-run-progress-reporter' +import { createFileUploader } from '../../piece-context/file-uploader' +import { createFlowsContext } from '../../piece-context/flows' +import { createContextStore } from '../../piece-context/store' +import { waitpointClient } from '../../piece-context/waitpoint-client' +import { utils } from '../../utils' +import { propsProcessor } from '../../variables/props-processor' +import { ActionContextRequest, CollectedHooks, ContextRequest, PieceRuntime, PropsContextRequest, TriggerContextRequest } from './piece-protocol' + +export async function buildContext({ piece, request }: BuildContextParams): Promise { + const hooks: CollectedHooks = { + hookResponse: { type: 'none', tags: [] }, + listeners: [], + } + const pending: Promise[] = [] + switch (request.kind) { + case 'action': + return { args: [await buildActionContext({ piece, request, hooks, pending })], hooks, pending } + case 'trigger': + return { args: [await buildTriggerContext({ piece, request, hooks })], hooks, pending } + case 'props': + return { args: [request.resolvedInput, buildPropsContext(request)], hooks, pending } + } +} + +async function buildActionContext({ piece, request, hooks, pending }: ActionParams): Promise { + const { runtime, stepName, actionName } = request + const action = piece.getAction(actionName) + if (isNil(action)) { + throw new EngineGenericError('ActionNotFoundError', `Action not found, actionName=${actionName}`) + } + const propsValue = await processProps({ request, props: action.props, requireAuth: action.requireAuth, piece }) + + const context: ActionContext = { + executionType: request.executionType, + resumePayload: request.resumePayload!, + store: createContextStore({ + apiUrl: runtime.internalApiUrl, + prefix: '', + flowId: runtime.flowId, + engineToken: runtime.engineToken, + }), + output: runtime.actionRunMode + ? { update: async (): Promise => Promise.resolve() } + : flowRunProgressReporter.createOutputContext(runtime), + flows: createFlowsContext({ + engineToken: runtime.engineToken, + internalApiUrl: runtime.internalApiUrl, + flowId: runtime.flowId, + flowVersionId: runtime.flowVersionId, + }), + step: { name: stepName }, + auth: propsValue[AUTHENTICATION_PROPERTY_NAME], + files: createFileUploader({ apiUrl: runtime.internalApiUrl, engineToken: runtime.engineToken }), + server: { + token: runtime.engineToken, + apiUrl: runtime.internalApiUrl, + publicUrl: runtime.publicApiUrl, + }, + propsValue, + tags: createTagsManager(hooks), + connections: createConnections({ runtime, target: 'actions', hooks }), + run: { + id: runtime.flowRunId, + stop: (request?: StopHookParams) => { + hooks.hookResponse = { ...hooks.hookResponse, type: 'stopped', response: request ?? { response: {} } } + }, + respond: (request?: StopHookParams) => { + hooks.hookResponse = { ...hooks.hookResponse, type: 'respond', response: request ?? { response: {} } } + }, + createWaitpoint: createWaitpointHook({ runtime, stepName, hooks, pending }), + waitForWaitpoint: () => { + assertCanSuspend(runtime) + hooks.hookResponse = { ...hooks.hookResponse, type: 'paused' } + }, + }, + project: createProjectContext(runtime), + } + + return backwardCompatabilityContextUtils.makeActionContextBackwardCompatible({ + contextVersion: runtime.contextVersion, + context, + }) +} + +async function buildTriggerContext({ piece, request, hooks }: TriggerParams): Promise { + const { runtime, stepName } = request + const trigger = piece.getTrigger(stepName) + if (isNil(trigger)) { + throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, stepName=${stepName}`) + } + const propsValue = await processProps({ request, props: trigger.props, requireAuth: trigger.requireAuth, piece }) + + return { + store: createContextStore({ + apiUrl: runtime.internalApiUrl, + prefix: request.storePrefix, + flowId: runtime.flowId, + engineToken: runtime.engineToken, + }), + step: { name: stepName }, + app: { + createListeners: ({ events, identifierKey, identifierValue }: { events: string[], identifierKey: string, identifierValue: string }): void => { + hooks.listeners.push({ events, identifierValue, identifierKey }) + }, + }, + setSchedule: (scheduleRequest: SetScheduleRequest) => { + hooks.scheduleOptions = parseSchedule(scheduleRequest) + }, + flows: createFlowsContext({ + engineToken: runtime.engineToken, + internalApiUrl: runtime.internalApiUrl, + flowId: runtime.flowId, + flowVersionId: runtime.flowVersionId, + }), + webhookUrl: request.webhookUrl, + isRepublish: request.isRepublish, + auth: propsValue[AUTHENTICATION_PROPERTY_NAME], + propsValue, + payload: request.payload ?? {}, + run: { id: runtime.flowRunId }, + project: createProjectContext(runtime), + server: { + token: runtime.engineToken, + apiUrl: runtime.internalApiUrl, + publicUrl: runtime.publicApiUrl, + }, + connections: createConnections({ runtime, target: 'triggers', hooks }), + ...(request.includeFiles ? { files: createFileUploader({ apiUrl: runtime.internalApiUrl, engineToken: runtime.engineToken }) } : {}), + } +} + +function buildPropsContext({ runtime, stepName, searchValue }: PropsContextRequest): unknown { + return { + searchValue, + server: { + token: runtime.engineToken, + apiUrl: runtime.internalApiUrl, + publicUrl: runtime.publicApiUrl, + }, + project: createProjectContext(runtime), + flows: createFlowsContext({ + engineToken: runtime.engineToken, + internalApiUrl: runtime.internalApiUrl, + flowId: runtime.flowId, + flowVersionId: runtime.flowVersionId, + }), + step: { name: stepName }, + connections: createConnections({ + runtime, + target: 'properties', + hooks: { hookResponse: { type: 'none', tags: [] }, listeners: [] }, + }), + } +} + +async function processProps({ request, props, requireAuth, piece }: ProcessPropsParams): Promise> { + const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators( + request.resolvedInput, + props, + piece.auth, + requireAuth, + request.propertySettings, + ) + if (Object.keys(errors).length > 0) { + throw new Error(JSON.stringify(errors, null, 2)) + } + return processedInput +} + +function createConnections({ runtime, target, hooks }: { runtime: PieceRuntime, target: 'actions' | 'triggers' | 'properties', hooks: CollectedHooks }): ReturnType { + return utils.createConnectionManager({ + apiUrl: runtime.internalApiUrl, + projectId: runtime.projectId, + engineToken: runtime.engineToken, + target, + hookResponse: hooks.hookResponse, + contextVersion: runtime.contextVersion, + pieceName: runtime.pieceName, + }) +} + +function createProjectContext(runtime: PieceRuntime): { id: string, externalId: () => Promise } { + return { + id: runtime.projectId, + externalId: async () => { + const response = await retryFetch(`${runtime.internalApiUrl}v1/worker/project`, { + headers: { Authorization: `Bearer ${runtime.engineToken}` }, + }) + const project = await response.json() + return isObject(project) && typeof project.externalId === 'string' ? project.externalId : undefined + }, + } +} + +function createTagsManager(hooks: CollectedHooks): TagsManager { + return { + add: async ({ name }: { name: string }): Promise => { + hooks.hookResponse.tags.push(name) + }, + } +} + +function createWaitpointHook({ runtime, stepName, hooks, pending }: WaitpointHookParams): CreateWaitpointHook { + return (params: CreateWaitpointParams) => { + const created = createWaitpoint({ runtime, stepName, hooks, params }) + pending.push(created) + return created + } +} + +async function createWaitpoint({ runtime, stepName, hooks, params }: SubmitWaitpointParams): Promise { + assertCanSuspend(runtime) + assertDelayWithinTimeout(params.resumeDateTime) + if (!isNil(params.responseToSend)) { + hooks.hookResponse = { ...hooks.hookResponse, responseToSend: params.responseToSend } + } + const result = await waitpointClient.create({ + apiUrl: runtime.internalApiUrl, + engineToken: runtime.engineToken, + flowRunId: runtime.flowRunId, + projectId: runtime.projectId, + stepName, + type: params.type, + version: params.version ?? 'V1', + resumeDateTime: params.resumeDateTime, + responseToSend: params.responseToSend, + workerHandlerId: runtime.workerHandlerId, + httpRequestId: runtime.httpRequestId, + }) + return { + ...result, + buildResumeUrl: ({ queryParams, sync }) => { + const url = new URL(`${result.resumeUrl}${sync ? '/sync' : ''}`) + url.search = new URLSearchParams(queryParams).toString() + return url.toString() + }, + } +} + +function parseSchedule(request: SetScheduleRequest): ScheduleOptions { + if ('intervalMs' in request) { + const parsed = ScheduleOptions.safeParse({ type: TriggerSourceScheduleType.INTERVAL, intervalMs: request.intervalMs }) + if (!parsed.success) { + throw new InvalidScheduleIntervalError(request.intervalMs) + } + return parsed.data + } + if (!isValidCron(request.cronExpression)) { + throw new InvalidCronExpressionError(request.cronExpression) + } + return { + type: TriggerSourceScheduleType.CRON_EXPRESSION, + cronExpression: request.cronExpression, + timezone: request.timezone ?? 'UTC', + } +} + +function assertCanSuspend(runtime: PieceRuntime): void { + if (runtime.actionRunMode) { + throw new Error('This action pauses the run (waitpoint) and can only run inside a flow, not as a action run.') + } +} + +function assertDelayWithinTimeout(resumeDateTime?: string): void { + if (isNil(resumeDateTime)) { + return + } + if (dayjs(resumeDateTime).diff(dayjs(), 'days') > AP_PAUSED_FLOW_TIMEOUT_DAYS) { + throw new PausedFlowTimeoutError(undefined, AP_PAUSED_FLOW_TIMEOUT_DAYS) + } +} + +const AP_PAUSED_FLOW_TIMEOUT_DAYS = Number(process.env.AP_PAUSED_FLOW_TIMEOUT_DAYS) + + +type BuildContextParams = { + piece: Piece + request: ContextRequest +} + +type ActionParams = { + piece: Piece + request: ActionContextRequest + hooks: CollectedHooks + pending: Promise[] +} + +type TriggerParams = { + piece: Piece + request: TriggerContextRequest + hooks: CollectedHooks +} + +type ProcessPropsParams = { + request: ActionContextRequest | TriggerContextRequest + props: Parameters[1] + requireAuth: boolean + piece: Piece +} + +type WaitpointHookParams = { + runtime: PieceRuntime + stepName: string + hooks: CollectedHooks + pending: Promise[] +} + +type SubmitWaitpointParams = { + runtime: PieceRuntime + stepName: string + hooks: CollectedHooks + params: CreateWaitpointParams +} + +export type BuiltContext = { + args: unknown[] + hooks: CollectedHooks + pending: Promise[] +} diff --git a/packages/server/engine/src/lib/core/piece/piece-path.ts b/packages/server/engine/src/lib/core/piece/piece-path.ts new file mode 100644 index 000000000000..043d0593fac3 --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-path.ts @@ -0,0 +1,142 @@ +import fs from 'fs/promises' +import path from 'path' +import { isNil } from '@activepieces/core-utils' +import { EngineGenericError, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared' +import { utils } from '../../utils' +import { PieceRef } from './piece-runner' + +export const piecePath = { + resolve: async ({ pieceName, pieceVersion, devPieces }: PieceRef): Promise => { + const packageName = getPackageAlias({ pieceName, pieceVersion, devPieces }) + const piecePath = devPieces.includes(getPieceNameFromAlias(packageName)) + ? await findInDistFolder(packageName) + : await traverseAllParentFoldersToFindPiece(packageName) + if (isNil(piecePath)) { + throw new EngineGenericError('PieceNotFoundError', `Piece not found for package: ${packageName}`) + } + return piecePath + }, + +} + +function getPackageAlias({ pieceName, pieceVersion, devPieces }: PieceRef): string { + if (devPieces.includes(getPieceNameFromAlias(pieceName))) { + return pieceName + } + + return getPackageAliasForPiece({ + pieceName, + pieceVersion, + }) +} + +async function findInDistFolder(packageName: string): Promise { + const sourcePiecesPath = path.resolve('packages/pieces') + if (!await utils.folderExists(sourcePiecesPath)) { + return null + } + const distPackageJsonPaths = await findDistPackageJsonFiles(sourcePiecesPath) + for (const packageJsonPath of distPackageJsonPaths) { + const { data: result } = await utils.tryCatchAndThrowOnEngineError(async () => { + const content = await fs.readFile(packageJsonPath, 'utf-8') + const packageJson = JSON.parse(content) + if (packageJson.name === packageName) { + return path.join(path.dirname(packageJsonPath), 'src', 'index.js') + } + return null + }) + if (result) { + return result + } + } + return null +} + +async function findDistPackageJsonFiles(dirPath: string): Promise { + const results: string[] = [] + const ignoredDirs = ['node_modules', '.turbo', 'framework', 'common'] + + async function scanDir(currentPath: string): Promise { + const items = await fs.readdir(currentPath, { withFileTypes: true }) + for (const item of items) { + if (!item.isDirectory() || ignoredDirs.includes(item.name)) { + continue + } + const fullPath = path.join(currentPath, item.name) + if (item.name === 'dist') { + const pkgJson = path.join(fullPath, 'package.json') + if (await utils.folderExists(pkgJson)) { + results.push(pkgJson) + } + } + else { + await scanDir(fullPath) + } + } + } + + await scanDir(dirPath) + return results +} + + +async function traverseAllParentFoldersToFindPiece(packageName: string): Promise { + const trimmedName = trimVersionFromAlias(packageName) + const customPaths = (process.env.AP_CUSTOM_PIECES_PATHS ?? '').split(':').filter(Boolean) + for (const customPath of customPaths) { + const entry = await resolveInstalledPieceEntry(path.resolve(customPath, 'pieces', packageName), trimmedName) + if (!isNil(entry)) { + return entry + } + } + + const rootDir = path.parse(__dirname).root + let currentDir = __dirname + const maxIterations = currentDir.split(path.sep).length + for (let i = 0; i < maxIterations; i++) { + const entry = await resolveInstalledPieceEntry(path.resolve(currentDir, 'pieces', packageName), trimmedName) + if (!isNil(entry)) { + return entry + } + + const parentDir = path.dirname(currentDir) + if (parentDir === currentDir || currentDir === rootDir) { + break + } + currentDir = parentDir + } + return null +} + +// A piece entry is resolved from its package.json "main" (defaulting to src/index.js). +// Registry/dev installs keep the package nested in node_modules; a packed-archive bundle is +// extracted straight to the install-folder root. Try the nested package first, then the root. +async function resolveInstalledPieceEntry(pieceFolder: string, trimmedName: string): Promise { + const packageDir = path.join(pieceFolder, 'node_modules', trimmedName) + if (await utils.folderExists(packageDir)) { + return resolveEntryFromPackageDir(packageDir) + } + // Only return an entry that actually exists: a half-installed registry folder also has a + // stub package.json (no "main") at this point, for which resolveEntryFromPackageDir would + // otherwise return a non-existent src/index.js — fall through to a clean PieceNotFoundError. + const rootManifest = path.join(pieceFolder, 'package.json') + if (await utils.folderExists(rootManifest)) { + const rootEntry = await resolveEntryFromPackageDir(pieceFolder) + if (await utils.folderExists(rootEntry)) { + return rootEntry + } + } + return null +} + +async function resolveEntryFromPackageDir(packageDir: string): Promise { + const { data: mainEntry } = await utils.tryCatchAndThrowOnEngineError(async () => { + const packageJson = JSON.parse(await fs.readFile(path.join(packageDir, 'package.json'), 'utf-8')) + if (isNil(packageJson.main)) { + return null + } + const resolved = path.join(packageDir, packageJson.main) + return await utils.folderExists(resolved) ? resolved : null + }) + return mainEntry ?? path.join(packageDir, 'src', 'index.js') +} diff --git a/packages/server/engine/src/lib/core/piece/piece-protocol.ts b/packages/server/engine/src/lib/core/piece/piece-protocol.ts new file mode 100644 index 000000000000..07d0143382c6 --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-protocol.ts @@ -0,0 +1,160 @@ +import { isNil, isObject } from '@activepieces/core-utils' +import { ContextVersion, PieceMetadata } from '@activepieces/pieces-framework' +import { ExecutionError, ExecutionErrorType, ExecutionType, PropertySettings, ResumePayload, ScheduleOptions } from '@activepieces/shared' +import { HookResponse } from '../../utils' + +export const pieceProtocol = { + toTransferable: (value: unknown): unknown => { + if (typeof value === 'function' || isThenable(value)) { + return undefined + } + if (Array.isArray(value)) { + return value.map((item) => pieceProtocol.toTransferable(item)) + } + if (!isObject(value) || Buffer.isBuffer(value) || value instanceof Date) { + return value + } + const entries = Object.entries(value) + .map(([key, item]) => [key, pieceProtocol.toTransferable(item)]) + .filter(([, item]) => item !== undefined) + return Object.fromEntries(entries) + }, + + serializeError: (error: unknown): SerializedError => { + if (!(error instanceof Error)) { + return { message: String(error) } + } + const details: Record = {} + for (const key of [...Object.keys(error), ...ERROR_DETAIL_KEYS]) { + const { data } = readJsonSafe(() => Reflect.get(error, key)) + if (data !== undefined) { + details[key] = data + } + } + return { + ...details, + message: error.message, + name: error.name === 'Error' ? error.constructor.name : error.name, + stack: error.stack, + type: error instanceof ExecutionError ? error.type : undefined, + } + }, + + deserializeError: ({ message, name, stack, type, ...details }: SerializedError): Error => { + if (!isNil(type)) { + return new ExecutionError(name ?? 'ExecutionError', message, type) + } + const error = Object.assign(new Error(message), details) + error.name = name ?? error.name + error.stack = stack ?? error.stack + return error + }, +} + +function isThenable(value: unknown): boolean { + return isObject(value) && typeof value.then === 'function' +} + +function readJsonSafe(read: () => unknown): { data: unknown } { + try { + return { data: JSON.parse(JSON.stringify(read())) } + } + catch { + return { data: undefined } + } +} + +const ERROR_DETAIL_KEYS = ['response', 'request', 'status', 'headers', 'body', 'error'] + +type PieceIdentity = { + piecePath: string + pieceName: string + pieceVersion: string +} + +export type PieceRuntime = { + internalApiUrl: string + publicApiUrl: string + engineToken: string + projectId: string + flowId: string + flowVersionId: string + flowRunId: string + pieceName: string + contextVersion?: ContextVersion + actionRunMode: boolean + workerHandlerId?: string + httpRequestId?: string +} + +export type ActionContextRequest = { + kind: 'action' + runtime: PieceRuntime + actionName: string + stepName: string + resolvedInput: Record + propertySettings: Record + executionType: ExecutionType + resumePayload?: ResumePayload +} + +export type TriggerContextRequest = { + kind: 'trigger' + runtime: PieceRuntime + stepName: string + resolvedInput: Record + propertySettings: Record + payload: unknown + storePrefix: string + includeFiles: boolean + webhookUrl?: string + isRepublish?: boolean +} + +export type PropsContextRequest = { + kind: 'props' + runtime: PieceRuntime + stepName: string + resolvedInput: Record + searchValue?: string +} + +export type ContextRequest = ActionContextRequest | TriggerContextRequest | PropsContextRequest + +export type PieceDescription = { + metadata: DescribedMetadata + functionPaths: string[] + hasPath: (path: string[]) => boolean +} + +type DescribedMetadata = Omit & { + i18n?: PieceMetadata['i18n'] +} + +type AppListener = { + events: string[] + identifierValue: string + identifierKey: string +} + +export type CollectedHooks = { + hookResponse: HookResponse + listeners: AppListener[] + scheduleOptions?: ScheduleOptions +} + +export type SerializedError = { + message: string + name?: string + stack?: string + type?: ExecutionErrorType + [key: string]: unknown +} + +export type ParentMessage = + | (PieceIdentity & { type: 'describe' }) + | (PieceIdentity & { type: 'call', path: string[], args: unknown[], context?: ContextRequest }) + +export type ChildMessage = + | { type: 'done', success: true, result: unknown, hooks?: CollectedHooks } + | { type: 'done', success: false, error: SerializedError } diff --git a/packages/server/engine/src/lib/core/piece/piece-runner.ts b/packages/server/engine/src/lib/core/piece/piece-runner.ts new file mode 100644 index 000000000000..3e8d412cb992 --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/piece-runner.ts @@ -0,0 +1,153 @@ +import { spawn } from 'node:child_process' +import path from 'node:path' +import { isNil, tryCatchSync } from '@activepieces/core-utils' +import { EngineGenericError, PieceMemoryLimitError } from '@activepieces/shared' +import { piecePath } from './piece-path' +import { ChildMessage, CollectedHooks, ContextRequest, ParentMessage, PieceDescription, pieceProtocol } from './piece-protocol' + +export const pieceRunner = { + describe: async (piece: PieceRef): Promise => { + const cacheKey = `${piece.pieceName}@${piece.pieceVersion}` + const cached = descriptions.get(cacheKey) + if (!isNil(cached)) { + return cached + } + const description = runInChildProcess({ piece, request: { type: 'describe' } }).then(({ result }) => toPieceDescription(result)) + descriptions.set(cacheKey, description) + description.catch(() => descriptions.delete(cacheKey)) + return description + }, + + call: async ({ piece, path: methodPath, args = [], context }: CallParams): Promise => { + return runInChildProcess({ piece, request: { type: 'call', path: methodPath, args, context } }) + }, +} + +async function runInChildProcess({ piece, request }: RunInChildProcessParams): Promise { + const entryPath = await piecePath.resolve(piece) + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [...process.execArgv, childEntryPath()], { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + serialization: 'advanced', + }) + let settled = false + let output = '' + + const settle = (apply: () => void): void => { + if (settled) { + return + } + settled = true + child.kill() + apply() + } + + child.stdout?.on('data', (data: Buffer) => { + output += data.toString() + console.log(data.toString().trimEnd()) + }) + child.stderr?.on('data', (data: Buffer) => { + output += data.toString() + console.error(data.toString().trimEnd()) + }) + + child.on('message', (message: ChildMessage) => { + settle(() => message.success + ? resolve({ result: message.result, hooks: message.hooks }) + : reject(pieceProtocol.deserializeError(message.error))) + }) + + child.on('close', (code, signal) => { + settle(() => reject(toExitError({ code, signal, output }))) + }) + + child.on('error', (error) => { + settle(() => reject(new EngineGenericError('PieceProcessError', withOutput(error.message, output)))) + }) + + const identity = { piecePath: entryPath, pieceName: piece.pieceName, pieceVersion: piece.pieceVersion } + const { error: sendError } = tryCatchSync(() => { + const message: ParentMessage = request.type === 'describe' + ? { ...identity, type: 'describe' } + : { ...identity, type: 'call', path: request.path, args: request.args, context: request.context } + child.send(message) + }) + if (sendError) { + settle(() => reject(new EngineGenericError('PieceArgumentsNotSerializableError', sendError.message))) + } + }) +} + +function toPieceDescription(value: unknown): PieceDescription { + const metadata = Reflect.get(Object(value), 'metadata') + const functionPaths = Reflect.get(Object(value), 'functionPaths') + if (isNil(metadata) || !Array.isArray(functionPaths)) { + throw new EngineGenericError('PieceDescriptionInvalidError', 'Piece process returned an unexpected description') + } + return { + metadata, + functionPaths, + hasPath: (path: string[]) => functionPaths.includes(path.join('.')), + } +} + +export function toExitError({ code, signal, output }: ExitParams): Error { + if (isOutOfMemory({ code, signal, output })) { + return new PieceMemoryLimitError(heapLimitMb(), output.trim()) + } + return new EngineGenericError('PieceProcessExitedError', withOutput(`Piece process exited with code ${code} and signal ${signal}`, output)) +} + +// Mirrors the engine-side signatures in sandbox.ts: V8 saying it ran out of heap, or aborting, +// is unambiguous. A SIGKILL is ambiguous there because shutdown kills the engine the same way — +// here it is not, since the only kill we issue happens after the call has already settled. +function isOutOfMemory({ code, signal, output }: ExitParams): boolean { + return output.includes('JavaScript heap out of memory') + || code === 134 + || signal === 'SIGABRT' + || signal === 'SIGKILL' +} + +function heapLimitMb(): string | undefined { + return process.execArgv.find((arg) => arg.startsWith('--max-old-space-size='))?.split('=')[1] +} + +function withOutput(message: string, output: string): string { + return output.trim().length === 0 ? message : `${message}\n${output.trim()}` +} + +function childEntryPath(): string { + return process.env.AP_PIECE_CHILD_ENTRY ?? path.join(__dirname, 'piece-child.js') +} + +const descriptions = new Map>() + +type RunInChildProcessParams = { + piece: PieceRef + request: { type: 'describe' } | { type: 'call', path: string[], args: unknown[], context?: ContextRequest } +} + +type ExitParams = { + code: number | null + signal: NodeJS.Signals | null + output: string +} + +export type PieceRef = { + pieceName: string + pieceVersion: string + devPieces: string[] +} + +export type CallParams = { + piece: PieceRef + path: string[] + args?: unknown[] + context?: ContextRequest +} + +export type CallResult = { + result: unknown + hooks?: CollectedHooks +} diff --git a/packages/server/engine/src/lib/core/piece/trigger-runner.ts b/packages/server/engine/src/lib/core/piece/trigger-runner.ts new file mode 100644 index 000000000000..19e737c5c6fe --- /dev/null +++ b/packages/server/engine/src/lib/core/piece/trigger-runner.ts @@ -0,0 +1,206 @@ +import { assertEqual, isNil, isObject } from '@activepieces/core-utils' +import { PiecePropertyMap, StaticPropsValue, TriggerStrategy } from '@activepieces/pieces-framework' +import { EngineGenericError, EngineHttpResponse, ExecuteTriggerResponse, FlowTrigger, PieceTrigger, PropertySettings, TriggerHookType } from '@activepieces/shared' +import { EngineConstants, ResolvedExecuteTriggerOperation } from '../../handler/context/engine-constants' +import { FlowExecutorContext } from '../../handler/context/flow-execution-context' +import { buildRuntime } from '../../handler/piece-executor' +import { createPropsResolver } from '../../variables/props-resolver' +import { CollectedHooks, TriggerContextRequest } from './piece-protocol' +import { PieceRef, pieceRunner } from './piece-runner' + +export const triggerRunner = { + async executeOnStart({ trigger, constants, payload }: ExecuteOnStartParams): Promise { + const { pieceName, pieceVersion, triggerName, input, propertySettings } = (trigger as PieceTrigger).settings + assertTriggerName(triggerName) + + const piece: PieceRef = { pieceName, pieceVersion, devPieces: constants.devPieces } + const description = await pieceRunner.describe(piece) + if (!description.hasPath(['triggers', triggerName, 'onStart'])) { + return + } + await pieceRunner.call({ + piece, + path: ['triggers', triggerName, 'onStart'], + context: await buildTriggerContext({ + piece, + constants, + triggerName, + input, + propertySettings, + contextVersion: description.metadata.contextInfo?.version, + payload, + storePrefix: '', + includeFiles: false, + }), + }) + }, + + async executeTrigger({ params, constants }: ExecuteTriggerParams): Promise> { + const { pieceName, pieceVersion, triggerName, input, propertySettings } = (params.flowVersion.trigger as PieceTrigger).settings + assertTriggerName(triggerName) + + const piece: PieceRef = { pieceName, pieceVersion, devPieces: constants.devPieces } + const description = await pieceRunner.describe(piece) + const pieceTrigger = description.metadata.triggers[triggerName] + if (isNil(pieceTrigger)) { + throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, pieceName=${pieceName}, triggerName=${triggerName}`) + } + + const context = await buildTriggerContext({ + piece, + constants, + triggerName, + input, + propertySettings, + contextVersion: description.metadata.contextInfo?.version, + payload: params.triggerPayload, + storePrefix: params.test ? 'test' : '', + includeFiles: params.hookType === TriggerHookType.TEST || params.hookType === TriggerHookType.RUN, + webhookUrl: params.webhookUrl, + isRepublish: params.isRepublish, + }) + const runHook = async (methodName: string): Promise<{ result: unknown, hooks?: CollectedHooks }> => + pieceRunner.call({ piece, path: ['triggers', triggerName, methodName], context }) + + switch (params.hookType) { + case TriggerHookType.ON_DISABLE: { + await runHook('onDisable') + return {} + } + case TriggerHookType.ON_ENABLE: { + const { hooks } = await runHook('onEnable') + return { + listeners: hooks?.listeners ?? [], + scheduleOptions: pieceTrigger.type === TriggerStrategy.POLLING ? hooks?.scheduleOptions : undefined, + } + } + case TriggerHookType.RENEW: { + assertEqual(pieceTrigger.type, TriggerStrategy.WEBHOOK, 'triggerType', 'WEBHOOK') + await runHook('onRenew') + return {} + } + case TriggerHookType.HANDSHAKE: { + const { result } = await runHook('onHandshake') + return { response: toWebhookResponse(result) } + } + case TriggerHookType.TEST: { + const { result } = await runHook('test') + return { output: toItems(result) } + } + case TriggerHookType.RUN: { + if (pieceTrigger.type === TriggerStrategy.APP_WEBHOOK) { + await verifyAppWebhook({ piece, description, params, pieceName }) + } + const { result } = await runHook('run') + return { output: toItems(result) } + } + } + }, +} + +async function buildTriggerContext({ piece, constants, triggerName, input, propertySettings, contextVersion, payload, storePrefix, includeFiles, webhookUrl, isRepublish }: BuildTriggerContextParams): Promise { + const { resolvedInput } = await createPropsResolver({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + contextVersion, + stepNames: constants.stepNames, + pieceName: piece.pieceName, + }).resolve>({ + unresolvedInput: input, + executionState: FlowExecutorContext.empty(), + }) + + return { + kind: 'trigger', + runtime: buildRuntime({ constants, pieceName: piece.pieceName, contextVersion }), + stepName: triggerName, + resolvedInput, + propertySettings, + payload, + storePrefix, + includeFiles, + webhookUrl, + isRepublish, + } +} + +async function verifyAppWebhook({ piece, description, params, pieceName }: VerifyAppWebhookParams): Promise { + if (!params.appWebhookUrl) { + throw new EngineGenericError('AppWebhookUrlNotAvailableError', `App webhook url is not available for piece name ${pieceName}`) + } + if (!params.webhookSecret) { + throw new EngineGenericError('WebhookSecretNotAvailableError', `Webhook secret is not available for piece name ${pieceName}`) + } + if (!description.hasPath(['events', 'verify'])) { + throw new Error('Webhook is not verified') + } + const { result } = await pieceRunner.call({ + piece, + path: ['events', 'verify'], + args: [{ + appWebhookUrl: params.appWebhookUrl, + payload: params.triggerPayload, + webhookSecret: params.webhookSecret, + }], + }) + if (result !== true) { + throw new Error('Webhook is not verified') + } +} + +function assertTriggerName(triggerName: string | undefined): asserts triggerName is string { + if (isNil(triggerName)) { + throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') + } +} + +function toItems(value: unknown): unknown[] { + if (!Array.isArray(value)) { + throw new EngineGenericError('TriggerOutputNotArrayError', `Trigger returned ${typeof value} instead of an array of items`) + } + return value +} + +function toWebhookResponse(value: unknown): { status: number, body?: unknown, headers?: Record } | undefined { + if (!isObject(value)) { + return undefined + } + return { + status: typeof value.status === 'number' ? value.status : 200, + body: value.body, + headers: EngineHttpResponse.shape.headers.safeParse(value.headers).data ?? {}, + } +} + +type ExecuteOnStartParams = { + trigger: FlowTrigger + constants: EngineConstants + payload: unknown +} + +type ExecuteTriggerParams = { + params: ResolvedExecuteTriggerOperation + constants: EngineConstants +} + +type BuildTriggerContextParams = { + piece: PieceRef + constants: EngineConstants + triggerName: string + input: unknown + propertySettings: Record + contextVersion: TriggerContextRequest['runtime']['contextVersion'] + payload: unknown + storePrefix: string + includeFiles: boolean + webhookUrl?: string + isRepublish?: boolean +} + +type VerifyAppWebhookParams = { + piece: PieceRef + description: Awaited> + params: ResolvedExecuteTriggerOperation + pieceName: string +} diff --git a/packages/server/engine/src/lib/handler/flow-executor.ts b/packages/server/engine/src/lib/handler/flow-executor.ts index 01ab3ebabb43..ad532a8537b4 100644 --- a/packages/server/engine/src/lib/handler/flow-executor.ts +++ b/packages/server/engine/src/lib/handler/flow-executor.ts @@ -2,9 +2,9 @@ import { performance } from 'node:perf_hooks' import { isNil } from '@activepieces/core-utils' import { EngineGenericError, ExecutionType, FlowAction, FlowActionType, FlowRunStatus, FlowTrigger, GenericStepOutput, StepOutputStatus } from '@activepieces/shared' import dayjs from 'dayjs' +import { triggerRunner } from '../core/piece/trigger-runner' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' import { loggingUtils } from '../helper/logging-utils' -import { triggerHelper } from '../helper/trigger-helper' import { BaseExecutor } from './base-executor' import { codeExecutor } from './code-executor' import { EngineConstants, ResolvedExecuteFlowOperation } from './context/engine-constants' @@ -49,7 +49,7 @@ export const flowExecutor = { void flowRunProgressReporter.backup().catch((err) => { console.error('[Progress] Initial payload upload failed', err) }) - await triggerHelper.executeOnStart(trigger, constants, input.triggerPayload) + await triggerRunner.executeOnStart({ trigger, constants, payload: input.triggerPayload }) await flowRunProgressReporter.sendUpdate({ engineConstants: constants, flowExecutorContext: executionState, diff --git a/packages/server/engine/src/lib/handler/piece-executor.ts b/packages/server/engine/src/lib/handler/piece-executor.ts index d02d0857d819..836b4a553059 100644 --- a/packages/server/engine/src/lib/handler/piece-executor.ts +++ b/packages/server/engine/src/lib/handler/piece-executor.ts @@ -1,22 +1,15 @@ -import { isNil } from '@activepieces/core-utils' -import { ActionContext, backwardCompatabilityContextUtils, CreateWaitpointHook, CreateWaitpointParams, CreateWaitpointResult, InputPropertyMap, PieceAuthProperty, PiecePropertyMap, RespondHook, RespondHookParams, StaticPropsValue, StopHook, StopHookParams, TagsManager, WaitForWaitpointHook } from '@activepieces/pieces-framework' -import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, ExecutionType, FlowActionType, FlowRunStatus, GenericStepOutput, PausedFlowTimeoutError, PieceAction, RespondResponse, StepOutputStatus } from '@activepieces/shared' -import dayjs from 'dayjs' +import { ActivepiecesError, ErrorCode, isNil } from '@activepieces/core-utils' +import { PiecePropertyMap, StaticPropsValue } from '@activepieces/pieces-framework' +import { EngineGenericError, ExecutionType, FlowActionType, FlowRunStatus, GenericStepOutput, PieceAction, RespondResponse, StepOutputStatus } from '@activepieces/shared' import { engineRunApi } from '../api/engine-run-api' +import { PieceRuntime } from '../core/piece/piece-protocol' +import { pieceRunner } from '../core/piece/piece-runner' import { continueIfFailureHandler, runWithExponentialBackoff } from '../helper/error-handling' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' -import { pieceLoader } from '../helper/piece-loader' -import { createFileUploader } from '../piece-context/file-uploader' -import { createFlowsContext } from '../piece-context/flows' -import { createContextStore } from '../piece-context/store' -import { waitpointClient } from '../piece-context/waitpoint-client' import { HookResponse, utils } from '../utils' -import { propsProcessor } from '../variables/props-processor' import { ActionHandler, BaseExecutor, failStep } from './base-executor' import { EngineConstants } from './context/engine-constants' -const AP_PAUSED_FLOW_TIMEOUT_DAYS = Number(process.env.AP_PAUSED_FLOW_TIMEOUT_DAYS) - export const pieceExecutor: BaseExecutor = { async handle({ action, @@ -40,44 +33,32 @@ const executeAction: ActionHandler = async ({ action, executionStat }) const { data: executionStateResult, error: executionStateError } = await utils.tryCatchAndThrowOnEngineError((async () => { - if (isNil(action.settings.actionName)) { + const { actionName, pieceName, pieceVersion, propertySettings } = action.settings + if (isNil(actionName)) { throw new EngineGenericError('ActionNameNotSetError', 'Action name is not set') } - const { pieceAction, piece } = await pieceLoader.getPieceAndActionOrThrow({ - pieceName: action.settings.pieceName, - pieceVersion: action.settings.pieceVersion, - actionName: action.settings.actionName, - devPieces: constants.devPieces, - }) + const piece = { pieceName, pieceVersion, devPieces: constants.devPieces } + const description = await pieceRunner.describe(piece) + if (isNil(description.metadata.actions[actionName])) { + throw new ActivepiecesError({ + code: ErrorCode.ENTITY_NOT_FOUND, + params: { + entityType: 'step', + entityId: actionName, + message: `Action not found for piece ${pieceName}@${pieceVersion}`, + extra: { pieceName, pieceVersion }, + }, + }) + } + const contextVersion = description.metadata.contextInfo?.version - const { resolvedInput, censoredInput } = await constants.getPropsResolver({ contextVersion: piece.getContextInfo?.().version, pieceName: action.settings.pieceName }).resolve>({ + const { resolvedInput, censoredInput } = await constants.getPropsResolver({ contextVersion, pieceName }).resolve>({ unresolvedInput: action.settings.input, executionState, }) - stepOutput.input = censoredInput - const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceAction.props, piece.auth, pieceAction.requireAuth, action.settings.propertySettings) - if (Object.keys(errors).length > 0) { - throw new Error(JSON.stringify(errors, null, 2)) - } - - - const params: { - hookResponse: HookResponse - } = { - hookResponse: { - type: 'none', - tags: [], - }, - } - const outputContext = constants.actionRunMode - ? { update: async (): Promise => { /* no-op: action runs have no live progress channel */ } } - : flowRunProgressReporter.createOutputContext({ - engineConstants: constants, - }) - const isPaused = executionState.isPaused({ stepName: action.name }) if (!isPaused) { await flowRunProgressReporter.sendUpdate({ @@ -86,69 +67,29 @@ const executeAction: ActionHandler = async ({ action, executionStat stepNameToUpdate: action.name, }) } - const context: ActionContext = { - executionType: isPaused ? ExecutionType.RESUME : ExecutionType.BEGIN, - resumePayload: constants.resumePayload!, - store: createContextStore({ - apiUrl: constants.internalApiUrl, - prefix: '', - flowId: constants.flowId, - engineToken: constants.engineToken, - }), - output: outputContext, - flows: createFlowsContext({ - engineToken: constants.engineToken, - internalApiUrl: constants.internalApiUrl, - flowId: constants.flowId, - flowVersionId: constants.flowVersionId, - }), - step: { - name: action.name, - }, - auth: processedInput[AUTHENTICATION_PROPERTY_NAME], - files: createFileUploader({ - apiUrl: constants.internalApiUrl, - engineToken: constants.engineToken, - }), - server: { - token: constants.engineToken, - apiUrl: constants.internalApiUrl, - publicUrl: constants.publicApiUrl, - }, - propsValue: processedInput, - tags: createTagsManager(params), - connections: utils.createConnectionManager({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - target: 'actions', - hookResponse: params.hookResponse, - contextVersion: piece.getContextInfo?.().version, - pieceName: action.settings.pieceName, - }), - run: { - id: constants.flowRunId, - stop: createStopHook(params), - respond: createRespondHook(params), - createWaitpoint: createWaitpointHook({ constants, stepName: action.name, hookParams: params }), - waitForWaitpoint: createWaitForWaitpointHook({ constants, hookParams: params }), - }, - project: { - id: constants.projectId, - externalId: constants.externalProjectId, + + const testSingleStepMode = !isNil(constants.stepNameToTest) + const useTestMethod = testSingleStepMode && description.hasPath(['actions', actionName, 'test']) + const { result: output, hooks } = await pieceRunner.call({ + piece, + path: ['actions', actionName, useTestMethod ? 'test' : 'run'], + context: { + kind: 'action', + runtime: buildRuntime({ constants, pieceName, contextVersion }), + actionName, + stepName: action.name, + resolvedInput, + propertySettings, + executionType: isPaused ? ExecutionType.RESUME : ExecutionType.BEGIN, + resumePayload: constants.resumePayload, }, - } - const backwardCompatibleContext = backwardCompatabilityContextUtils.makeActionContextBackwardCompatible({ - contextVersion: piece.getContextInfo?.().version, - context, }) - const testSingleStepMode = !isNil(constants.stepNameToTest) - const runMethodToExecute = (testSingleStepMode && !isNil(pieceAction.test)) ? pieceAction.test : pieceAction.run - const output = await runMethodToExecute(backwardCompatibleContext) - const newExecutionContext = executionState.addTags(params.hookResponse.tags) - const webhookResponse = getResponse(params.hookResponse) - const isSamePiece = constants.triggerPieceName === action.settings.pieceName + const hookResponse: HookResponse = hooks?.hookResponse ?? { type: 'none', tags: [] } + const newExecutionContext = executionState.addTags(hookResponse.tags) + + const webhookResponse = getResponse(hookResponse) + const isSamePiece = constants.triggerPieceName === pieceName if (!isNil(webhookResponse) && !isNil(constants.workerHandlerId) && !isNil(constants.httpRequestId) && isSamePiece) { await engineRunApi.sendFlowResponse({ apiUrl: constants.internalApiUrl, @@ -166,17 +107,17 @@ const executeAction: ActionHandler = async ({ action, executionStat } const stepEndTime = performance.now() - if (params.hookResponse.type === 'stopped') { - if (isNil(params.hookResponse.response)) { + if (hookResponse.type === 'stopped') { + if (isNil(hookResponse.response)) { throw new EngineGenericError('StopResponseNotSetError', 'Stop response is not set') } const succeeded = stepOutput.setOutput(output).setStatus(StepOutputStatus.SUCCEEDED).setDuration(stepEndTime - stepStartTime) return (await newExecutionContext.upsertStep(action.name, succeeded)).incrementStepsExecuted().setVerdict({ status: FlowRunStatus.SUCCEEDED, - stopResponse: (params.hookResponse.response as StopHookParams).response, + stopResponse: hookResponse.response.response, }) } - if (params.hookResponse.type === 'paused') { + if (hookResponse.type === 'paused') { const paused = stepOutput.setOutput(output).setStatus(StepOutputStatus.PAUSED).setDuration(stepEndTime - stepStartTime) return (await newExecutionContext.upsertStep(action.name, paused)) .incrementStepsExecuted() @@ -212,108 +153,25 @@ function getResponse(hookResponse: HookResponse): RespondResponse | undefined { } } -const createTagsManager = (hkParams: createTagsManagerParams): TagsManager => { +export function buildRuntime({ constants, pieceName, contextVersion }: BuildRuntimeParams): PieceRuntime { return { - add: async (params: addTagsParams): Promise => { - hkParams.hookResponse.tags.push(params.name) - }, - - } -} - -type addTagsParams = { - name: string -} - -type createTagsManagerParams = { - hookResponse: HookResponse -} - - -function createStopHook(params: CreateStopHookParams): StopHook { - return (req?: StopHookParams) => { - params.hookResponse = { - ...params.hookResponse, - type: 'stopped', - response: req ?? { response: {} }, - } - } -} -type CreateStopHookParams = { - hookResponse: HookResponse -} - -function createRespondHook(params: CreateRespondHookParams): RespondHook { - return (req?: RespondHookParams) => { - params.hookResponse = { - ...params.hookResponse, - type: 'respond', - response: req ?? { response: {} }, - } - } -} - -type CreateRespondHookParams = { - hookResponse: HookResponse -} - -function createWaitpointHook({ constants, stepName, hookParams }: { constants: EngineConstants, stepName: string, hookParams: { hookResponse: HookResponse } }): CreateWaitpointHook { - return (req: CreateWaitpointParams): Promise => { - assertActionRunCannotSuspend(constants) - return submitWaitpoint({ constants, stepName, hookParams, req }) - } -} - -async function submitWaitpoint({ constants, stepName, hookParams, req }: { constants: EngineConstants, stepName: string, hookParams: { hookResponse: HookResponse }, req: CreateWaitpointParams }): Promise { - assertDelayWithinTimeout(req.resumeDateTime) - if (!isNil(req.responseToSend)) { - hookParams.hookResponse = { ...hookParams.hookResponse, responseToSend: req.responseToSend } - } - const result = await waitpointClient.create({ - apiUrl: constants.internalApiUrl, + internalApiUrl: constants.internalApiUrl, + publicApiUrl: constants.publicApiUrl, engineToken: constants.engineToken, - flowRunId: constants.flowRunId, projectId: constants.projectId, - stepName, - type: req.type, - version: req.version ?? 'V1', - resumeDateTime: req.resumeDateTime, - responseToSend: req.responseToSend, + flowId: constants.flowId, + flowVersionId: constants.flowVersionId, + flowRunId: constants.flowRunId, + pieceName, + contextVersion, + actionRunMode: constants.actionRunMode, workerHandlerId: constants.workerHandlerId ?? undefined, httpRequestId: constants.httpRequestId ?? undefined, - }) - return { - ...result, - buildResumeUrl: (params: { queryParams: Record, sync?: boolean }): string => { - const url = new URL(`${result.resumeUrl}${params.sync ? '/sync' : ''}`) - url.search = new URLSearchParams(params.queryParams).toString() - return url.toString() - }, - } -} - -function createWaitForWaitpointHook({ constants, hookParams }: { constants: EngineConstants, hookParams: { hookResponse: HookResponse } }): WaitForWaitpointHook { - return (_waitpointId: string) => { - assertActionRunCannotSuspend(constants) - hookParams.hookResponse = { - ...hookParams.hookResponse, - type: 'paused', - } - } -} - -function assertActionRunCannotSuspend(constants: EngineConstants): void { - if (constants.actionRunMode) { - throw new Error('This action pauses the run (waitpoint) and can only run inside a flow, not as a action run.') } } -function assertDelayWithinTimeout(resumeDateTime?: string): void { - if (isNil(resumeDateTime)) { - return - } - const diffInDays = dayjs(resumeDateTime).diff(dayjs(), 'days') - if (diffInDays > AP_PAUSED_FLOW_TIMEOUT_DAYS) { - throw new PausedFlowTimeoutError(undefined, AP_PAUSED_FLOW_TIMEOUT_DAYS) - } +type BuildRuntimeParams = { + constants: EngineConstants + pieceName: string + contextVersion?: PieceRuntime['contextVersion'] } diff --git a/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts b/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts index 618c1920ee5c..181d112fe854 100644 --- a/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts +++ b/packages/server/engine/src/lib/helper/flow-run-progress-reporter.ts @@ -70,17 +70,16 @@ export const flowRunProgressReporter = { }) }) }, - createOutputContext: (params: CreateOutputContextParams): OutputContext => { - const { engineConstants } = params + createOutputContext: ({ internalApiUrl, engineToken, projectId, flowRunId }: CreateOutputContextParams): OutputContext => { return { update: async (params: { data: unknown }) => { // Streaming output is best-effort — a failed push must never fail the run. const { error } = await tryCatch(() => engineRunApi.updateStepProgress({ - apiUrl: engineConstants.internalApiUrl, - engineToken: engineConstants.engineToken, + apiUrl: internalApiUrl, + engineToken, request: { - projectId: engineConstants.projectId, - runId: engineConstants.flowRunId, + projectId, + runId: flowRunId, output: params.data, }, })) @@ -244,7 +243,10 @@ type UpdateStepProgressParams = { } type CreateOutputContextParams = { - engineConstants: EngineConstants + internalApiUrl: string + engineToken: string + projectId: string + flowRunId: string } type ExtractStepResponse = { diff --git a/packages/server/engine/src/lib/helper/piece-helper.ts b/packages/server/engine/src/lib/helper/piece-helper.ts deleted file mode 100644 index 30df4dfb5841..000000000000 --- a/packages/server/engine/src/lib/helper/piece-helper.ts +++ /dev/null @@ -1,366 +0,0 @@ -import path from 'path' -import { isNil } from '@activepieces/core-utils' -import { - DropdownProperty, - DynamicProperties, - ExecutePropsResult, - getAuthPropertyForValue, - MultiSelectDropdownProperty, - PieceAuthProperty, - PieceMetadata, - PiecePropertyMap, - pieceTranslation, - PropertyType, - StaticPropsValue } from '@activepieces/pieces-framework' -import { AppConnectionType, AppConnectionValue, EngineGenericError, ExecuteExtractPieceMetadata, ExecutePropsOptions, ExecuteRefreshTokenAuthOperation, ExecuteRefreshTokenAuthResponse, ExecuteResolveConnectionIdentifierOperation, ExecuteResolveConnectionIdentifierResponse, ExecuteValidateAuthOperation, ExecuteValidateAuthResponse } from '@activepieces/shared' -import { EngineConstants } from '../handler/context/engine-constants' - -const DEFAULT_REFRESH_EXPIRES_IN_SECONDS = 3300 -import { testExecutionContext } from '../handler/context/test-execution-context' -import { createFlowsContext } from '../piece-context/flows' -import { utils } from '../utils' -import { createPropsResolver } from '../variables/props-resolver' -import { dynamicPropKeys } from './dynamic-prop-keys' -import { pieceLoader } from './piece-loader' - -export const pieceHelper = { - async executeProps( operation: ExecutePropsParams): Promise> { - const constants = EngineConstants.fromExecutePropertyInput(operation) - const executionState = await testExecutionContext.stateFromFlowVersion({ - apiUrl: operation.internalApiUrl, - flowVersion: operation.flowVersion, - projectId: operation.projectId, - engineToken: operation.engineToken, - sampleData: operation.sampleData, - engineConstants: constants, - }) - const { property, piece } = await pieceLoader.getPropOrThrow({ pieceName: operation.pieceName, pieceVersion: operation.pieceVersion, actionOrTriggerName: operation.actionOrTriggerName, propertyName: operation.propertyName, devPieces: EngineConstants.DEV_PIECES }) - - if (property.type !== PropertyType.DROPDOWN && property.type !== PropertyType.MULTI_SELECT_DROPDOWN && property.type !== PropertyType.DYNAMIC) { - throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property.type} for ${property.displayName}`) - } - const { data: executePropsResult, error: executePropsError } = await utils.tryCatchAndThrowOnEngineError((async (): Promise> => { - const { resolvedInput } = await createPropsResolver({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - contextVersion: piece.getContextInfo?.().version, - stepNames: constants.stepNames, - pieceName: operation.pieceName, - }).resolve< - StaticPropsValue - >({ - unresolvedInput: operation.input, - executionState, - }) - const ctx = { - searchValue: operation.searchValue, - server: { - token: constants.engineToken, - apiUrl: constants.internalApiUrl, - publicUrl: operation.publicApiUrl, - }, - project: { - id: constants.projectId, - externalId: constants.externalProjectId, - }, - flows: createFlowsContext(constants), - step: { - name: operation.actionOrTriggerName, - }, - connections: utils.createConnectionManager({ - projectId: constants.projectId, - engineToken: constants.engineToken, - apiUrl: constants.internalApiUrl, - target: 'properties', - contextVersion: piece.getContextInfo?.().version, - pieceName: operation.pieceName, - }), - } - - switch (property.type) { - case PropertyType.DYNAMIC: { - const dynamicProperty = property as DynamicProperties - const props = await dynamicProperty.props(resolvedInput, ctx) - return { - type: PropertyType.DYNAMIC, - options: dynamicPropKeys.escapePropsKeys(props), - } - } - case PropertyType.MULTI_SELECT_DROPDOWN: { - const multiSelectProperty = property as MultiSelectDropdownProperty< - unknown, - boolean - > - const options = await multiSelectProperty.options(resolvedInput, ctx) - return { - type: PropertyType.MULTI_SELECT_DROPDOWN, - options, - } - } - case PropertyType.DROPDOWN: { - const dropdownProperty = property as DropdownProperty - const options = await dropdownProperty.options(resolvedInput, ctx) - return { - type: PropertyType.DROPDOWN, - options, - } - } - default: { - throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property}`) - } - } - })) - - if (executePropsError) { - console.error(executePropsError) - return { - type: property.type, - options: { - disabled: true, - options: [], - placeholder: 'Throws an error, reconnect or refresh the page', - }, - } - } - - return executePropsResult - }, - - async executeValidateAuth( - { params, devPieces }: { params: ExecuteValidateAuthOperation, devPieces: string[] }, - ): Promise { - const { piece: piecePackage } = params - - const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) - const server = buildServerContext(params) - return validateAuth({ - authValue: params.auth, - pieceAuth: piece.auth, - server, - }) - - }, - - async executeResolveConnectionIdentifier( - { params, devPieces }: { params: ExecuteResolveConnectionIdentifierOperation, devPieces: string[] }, - ): Promise { - const { piece: piecePackage } = params - - const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) - const server = buildServerContext(params) - return resolveConnectionIdentifier({ - authValue: params.auth, - connectionType: params.connectionType, - pieceAuth: piece.auth, - server, - }) - }, - - async executeRefreshTokenAuth( - { params, devPieces }: { params: ExecuteRefreshTokenAuthOperation, devPieces: string[] }, - ): Promise { - const { piece: piecePackage } = params - - const piece = await pieceLoader.loadPieceOrThrow({ pieceName: piecePackage.pieceName, pieceVersion: piecePackage.pieceVersion, devPieces }) - - if (params.auth.type !== AppConnectionType.CUSTOM_AUTH) { - return { skipped: true } - } - - const pieceAuth = getAuthPropertyForValue({ authValueType: params.auth.type, pieceAuth: piece.auth }) - - if (isNil(pieceAuth) || pieceAuth.type !== PropertyType.CUSTOM_AUTH || isNil(pieceAuth.refresh)) { - return { skipped: true } - } - - const server = buildServerContext(params) - const result = await pieceAuth.refresh.generate({ - auth: params.auth.props, - server, - }) - - const expiresIn = result.expires_in ?? pieceAuth.refresh.defaultExpiresIn ?? DEFAULT_REFRESH_EXPIRES_IN_SECONDS - - return { - skipped: false, - access_token: result.access_token, - expires_in: expiresIn, - } - }, - - async extractPieceMetadata({ devPieces, params }: { devPieces: string[], params: ExecuteExtractPieceMetadata }): Promise { - const { pieceName, pieceVersion } = params - const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) - const pieceAlias = pieceLoader.getPackageAlias({ pieceName, pieceVersion, devPieces }) - const pieceIndexPath = await pieceLoader.getPiecePath({ packageName: pieceAlias, devPieces }) - const pieceDistRoot = path.dirname(path.dirname(pieceIndexPath)) - const i18n = await pieceTranslation.initializeI18n(pieceDistRoot) - const fullMetadata = piece.metadata() - return { - ...fullMetadata, - name: pieceName, - version: pieceVersion, - authors: piece.authors, - i18n, - } - }, -} - -type ExecutePropsParams = Omit & { pieceName: string, pieceVersion: string } - - -function mismatchAuthTypeErrorMessage(pieceAuthType: PropertyType, connectionType: AppConnectionType): ExecuteValidateAuthResponse { - return { - valid: false, - error: `Connection value type does not match piece auth type: ${pieceAuthType} !== ${connectionType}`, - } -} - -const validateAuth = async ({ - server, - authValue, - pieceAuth, -}: ValidateAuthParams): Promise => { - if (isNil(pieceAuth)) { - return { - valid: true, - } - } - const usedPieceAuth = getAuthPropertyForValue({ - authValueType: authValue.type, - pieceAuth, - }) - - if (isNil(usedPieceAuth)) { - return { - valid: false, - error: 'No piece auth found for auth value', - } - } - if (isNil(usedPieceAuth.validate)) { - return { - valid: true, - } - } - - - switch (usedPieceAuth.type) { - case PropertyType.OAUTH2:{ - if (authValue.type !== AppConnectionType.OAUTH2 && authValue.type !== AppConnectionType.CLOUD_OAUTH2 && authValue.type !== AppConnectionType.PLATFORM_OAUTH2) { - return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) - } - return usedPieceAuth.validate({ - auth: authValue, - server, - }) - } - case PropertyType.BASIC_AUTH:{ - if (authValue.type !== AppConnectionType.BASIC_AUTH) { - return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) - } - return usedPieceAuth.validate({ - auth: authValue, - server, - }) - } - case PropertyType.SECRET_TEXT:{ - if (authValue.type !== AppConnectionType.SECRET_TEXT) { - return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) - } - return usedPieceAuth.validate({ - auth: authValue.secret_text, - server, - }) - } - case PropertyType.CUSTOM_AUTH:{ - if (authValue.type !== AppConnectionType.CUSTOM_AUTH) { - return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) - } - return usedPieceAuth.validate({ - auth: authValue.props, - server, - }) - } - case PropertyType.OIDC:{ - if (authValue.type !== AppConnectionType.OIDC) { - return mismatchAuthTypeErrorMessage(usedPieceAuth.type, authValue.type) - } - return usedPieceAuth.validate({ - auth: authValue.props, - server, - }) - } - default: { - throw new EngineGenericError('InvalidAuthTypeError', 'Invalid auth type') - } - } -} - -const resolveConnectionIdentifier = async ({ - server, - authValue, - connectionType, - pieceAuth, -}: ResolveConnectionIdentifierParams): Promise => { - if (isNil(pieceAuth)) { - return { identifier: undefined } - } - const usedPieceAuth = getAuthPropertyForValue({ - authValueType: connectionType, - pieceAuth, - }) - if (isNil(usedPieceAuth)) { - return { identifier: undefined } - } - switch (usedPieceAuth.type) { - case PropertyType.OAUTH2: { - if (!('access_token' in authValue)) { - return { identifier: undefined } - } - return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue, server }) } - } - case PropertyType.BASIC_AUTH: { - if (!('username' in authValue)) { - return { identifier: undefined } - } - return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue, server }) } - } - case PropertyType.SECRET_TEXT: { - if (!('secret_text' in authValue)) { - return { identifier: undefined } - } - return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue.secret_text, server }) } - } - case PropertyType.CUSTOM_AUTH: - case PropertyType.OIDC: { - if (!('props' in authValue)) { - return { identifier: undefined } - } - return { identifier: await usedPieceAuth.getConnectionIdentifier?.({ auth: authValue.props, server }) } - } - default: { - return { identifier: undefined } - } - } -} - -type ValidateAuthParams = { - server: { - apiUrl: string - publicUrl: string - } - authValue: AppConnectionValue - pieceAuth: PieceAuthProperty | PieceAuthProperty[] | undefined -} - -type ResolveConnectionIdentifierParams = ValidateAuthParams & { - connectionType: AppConnectionType -} - -function buildServerContext({ internalApiUrl, publicApiUrl }: { internalApiUrl: string, publicApiUrl: string }) { - return { - apiUrl: internalApiUrl.endsWith('/') ? internalApiUrl : internalApiUrl + '/', - publicUrl: publicApiUrl, - } -} \ No newline at end of file diff --git a/packages/server/engine/src/lib/helper/piece-loader.ts b/packages/server/engine/src/lib/helper/piece-loader.ts deleted file mode 100644 index 6a67607ff74b..000000000000 --- a/packages/server/engine/src/lib/helper/piece-loader.ts +++ /dev/null @@ -1,282 +0,0 @@ -import fs from 'fs/promises' -import path from 'path' -import { ActivepiecesError, ErrorCode, isNil } from '@activepieces/core-utils' -import { Action, Piece, PiecePropertyMap, Trigger } from '@activepieces/pieces-framework' -import { EngineGenericError, extractPieceFromModule, getPackageAliasForPiece, getPieceNameFromAlias, trimVersionFromAlias } from '@activepieces/shared' -import { utils } from '../utils' - -export const pieceLoader = { - loadPieceOrThrow: async ( - { pieceName, pieceVersion, devPieces }: LoadPieceParams, - ): Promise => { - const { data: piece, error: pieceError } = await utils.tryCatchAndThrowOnEngineError(async () => { - const packageName = pieceLoader.getPackageAlias({ - pieceName, - pieceVersion, - devPieces, - }) - const piecePath = await pieceLoader.getPiecePath({ packageName, devPieces }) - const module = await import(piecePath) - - const piece = extractPieceFromModule({ - module, - pieceName, - pieceVersion, - }) - - if (isNil(piece)) { - throw new EngineGenericError('PieceNotFoundError', `Piece not found for piece: ${pieceName}, pieceVersion: ${pieceVersion}`) - } - return piece - }) - if (pieceError) { - throw pieceError - } - return piece - }, - - getPieceAndTriggerOrThrow: async (params: GetPieceAndTriggerParams): Promise<{ piece: Piece, pieceTrigger: Trigger }> => { - const { pieceName, pieceVersion, triggerName, devPieces } = params - const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) - const trigger = piece.getTrigger(triggerName) - - if (trigger === undefined) { - throw new EngineGenericError('TriggerNotFoundError', `Trigger not found, pieceName=${pieceName}, triggerName=${triggerName}`) - } - - return { - piece, - pieceTrigger: trigger, - } - }, - - getPieceAndActionOrThrow: async (params: GetPieceAndActionParams): Promise<{ piece: Piece, pieceAction: Action }> => { - const { pieceName, pieceVersion, actionName, devPieces } = params - - const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) - const pieceAction = piece.getAction(actionName) - - if (isNil(pieceAction)) { - throw new ActivepiecesError({ - code: ErrorCode.ENTITY_NOT_FOUND, - params: { - entityType: 'step', - entityId: actionName, - message: `Action not found for piece ${pieceName}@${pieceVersion}`, - extra: { pieceName, pieceVersion }, - }, - }) - } - - return { - piece, - pieceAction, - } - }, - - getPropOrThrow: async ({ pieceName, pieceVersion, actionOrTriggerName, propertyName, devPieces }: GetPropParams) => { - const piece = await pieceLoader.loadPieceOrThrow({ pieceName, pieceVersion, devPieces }) - - const actionOrTrigger = piece.getAction(actionOrTriggerName) ?? piece.getTrigger(actionOrTriggerName) - - if (isNil(actionOrTrigger)) { - throw new ActivepiecesError({ - code: ErrorCode.ENTITY_NOT_FOUND, - params: { - entityType: 'step', - entityId: actionOrTriggerName, - message: `Step not found for piece ${pieceName}@${pieceVersion}`, - extra: { pieceName, pieceVersion }, - }, - }) - } - - const property = (actionOrTrigger.props as PiecePropertyMap)[propertyName] - - if (isNil(property)) { - throw new ActivepiecesError({ - code: ErrorCode.ENTITY_NOT_FOUND, - params: { - entityType: 'config', - entityId: propertyName, - message: `Config not found for step ${actionOrTriggerName} in piece ${pieceName}@${pieceVersion}`, - extra: { pieceName, pieceVersion, stepName: actionOrTriggerName }, - }, - }) - } - - return { property, piece } - }, - - getPackageAlias: ({ pieceName, pieceVersion, devPieces }: GetPackageAliasParams) => { - if (devPieces.includes(getPieceNameFromAlias(pieceName))) { - return pieceName - } - - return getPackageAliasForPiece({ - pieceName, - pieceVersion, - }) - }, - - getPiecePath: async ({ packageName, devPieces }: GetPiecePathParams): Promise => { - const piecePath = devPieces.includes(getPieceNameFromAlias(packageName)) - ? await findInDistFolder(packageName) - : await traverseAllParentFoldersToFindPiece(packageName) - if (isNil(piecePath)) { - throw new EngineGenericError('PieceNotFoundError', `Piece not found for package: ${packageName}`) - } - return piecePath - }, -} - -async function findInDistFolder(packageName: string): Promise { - const sourcePiecesPath = path.resolve('packages/pieces') - if (!await utils.folderExists(sourcePiecesPath)) { - return null - } - const distPackageJsonPaths = await findDistPackageJsonFiles(sourcePiecesPath) - for (const packageJsonPath of distPackageJsonPaths) { - const { data: result } = await utils.tryCatchAndThrowOnEngineError(async () => { - const content = await fs.readFile(packageJsonPath, 'utf-8') - const packageJson = JSON.parse(content) - if (packageJson.name === packageName) { - return path.join(path.dirname(packageJsonPath), 'src', 'index.js') - } - return null - }) - if (result) { - return result - } - } - return null -} - -async function findDistPackageJsonFiles(dirPath: string): Promise { - const results: string[] = [] - const ignoredDirs = ['node_modules', '.turbo', 'framework', 'common'] - - async function scanDir(currentPath: string): Promise { - const items = await fs.readdir(currentPath, { withFileTypes: true }) - for (const item of items) { - if (!item.isDirectory() || ignoredDirs.includes(item.name)) { - continue - } - const fullPath = path.join(currentPath, item.name) - if (item.name === 'dist') { - const pkgJson = path.join(fullPath, 'package.json') - if (await utils.folderExists(pkgJson)) { - results.push(pkgJson) - } - } - else { - await scanDir(fullPath) - } - } - } - - await scanDir(dirPath) - return results -} - - -async function traverseAllParentFoldersToFindPiece(packageName: string): Promise { - const trimmedName = trimVersionFromAlias(packageName) - const customPaths = (process.env.AP_CUSTOM_PIECES_PATHS ?? '').split(':').filter(Boolean) - for (const customPath of customPaths) { - const entry = await resolveInstalledPieceEntry(path.resolve(customPath, 'pieces', packageName), trimmedName) - if (!isNil(entry)) { - return entry - } - } - - const rootDir = path.parse(__dirname).root - let currentDir = __dirname - const maxIterations = currentDir.split(path.sep).length - for (let i = 0; i < maxIterations; i++) { - const entry = await resolveInstalledPieceEntry(path.resolve(currentDir, 'pieces', packageName), trimmedName) - if (!isNil(entry)) { - return entry - } - - const parentDir = path.dirname(currentDir) - if (parentDir === currentDir || currentDir === rootDir) { - break - } - currentDir = parentDir - } - return null -} - -// A piece entry is resolved from its package.json "main" (defaulting to src/index.js). -// Registry/dev installs keep the package nested in node_modules; a packed-archive bundle is -// extracted straight to the install-folder root. Try the nested package first, then the root. -async function resolveInstalledPieceEntry(pieceFolder: string, trimmedName: string): Promise { - const packageDir = path.join(pieceFolder, 'node_modules', trimmedName) - if (await utils.folderExists(packageDir)) { - return resolveEntryFromPackageDir(packageDir) - } - // Only return an entry that actually exists: a half-installed registry folder also has a - // stub package.json (no "main") at this point, for which resolveEntryFromPackageDir would - // otherwise return a non-existent src/index.js — fall through to a clean PieceNotFoundError. - const rootManifest = path.join(pieceFolder, 'package.json') - if (await utils.folderExists(rootManifest)) { - const rootEntry = await resolveEntryFromPackageDir(pieceFolder) - if (await utils.folderExists(rootEntry)) { - return rootEntry - } - } - return null -} - -async function resolveEntryFromPackageDir(packageDir: string): Promise { - const { data: mainEntry } = await utils.tryCatchAndThrowOnEngineError(async () => { - const packageJson = JSON.parse(await fs.readFile(path.join(packageDir, 'package.json'), 'utf-8')) - if (isNil(packageJson.main)) { - return null - } - const resolved = path.join(packageDir, packageJson.main) - return await utils.folderExists(resolved) ? resolved : null - }) - return mainEntry ?? path.join(packageDir, 'src', 'index.js') -} - -type GetPiecePathParams = { - packageName: string - devPieces: string[] -} - -type LoadPieceParams = { - pieceName: string - pieceVersion: string - devPieces: string[] -} - -type GetPieceAndTriggerParams = { - pieceName: string - pieceVersion: string - triggerName: string - devPieces: string[] -} - -type GetPieceAndActionParams = { - pieceName: string - pieceVersion: string - actionName: string - devPieces: string[] -} - -type GetPropParams = { - pieceName: string - pieceVersion: string - actionOrTriggerName: string - propertyName: string - devPieces: string[] -} - -type GetPackageAliasParams = { - pieceName: string - devPieces: string[] - pieceVersion: string -} - diff --git a/packages/server/engine/src/lib/helper/trigger-helper.ts b/packages/server/engine/src/lib/helper/trigger-helper.ts deleted file mode 100644 index 0d6417fd80a9..000000000000 --- a/packages/server/engine/src/lib/helper/trigger-helper.ts +++ /dev/null @@ -1,301 +0,0 @@ -import { assertEqual, isNil } from '@activepieces/core-utils' -import { PiecePropertyMap, SetScheduleRequest, StaticPropsValue, TriggerStrategy } from '@activepieces/pieces-framework' -import { AUTHENTICATION_PROPERTY_NAME, EngineGenericError, EventPayload, ExecuteTriggerResponse, FlowTrigger, InvalidCronExpressionError, InvalidScheduleIntervalError, PieceTrigger, PropertySettings, ScheduleOptions, TriggerHookType, TriggerSourceScheduleType } from '@activepieces/shared' -import { isValidCron } from 'cron-validator' -import { EngineConstants, ResolvedExecuteTriggerOperation } from '../handler/context/engine-constants' -import { FlowExecutorContext } from '../handler/context/flow-execution-context' -import { createFileUploader } from '../piece-context/file-uploader' -import { createFlowsContext } from '../piece-context/flows' -import { createContextStore } from '../piece-context/store' -import { utils } from '../utils' -import { propsProcessor } from '../variables/props-processor' -import { createPropsResolver } from '../variables/props-resolver' -import { pieceLoader } from './piece-loader' - -type Listener = { - events: string[] - identifierValue: string - identifierKey: string -} - -export const triggerHelper = { - async executeOnStart(trigger: FlowTrigger, constants: EngineConstants, payload: unknown) { - const { pieceName, pieceVersion, triggerName, input, propertySettings } = (trigger as PieceTrigger).settings - - if (isNil(triggerName)) { - throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') - } - - const { pieceTrigger, processedInput, piece } = await prepareTriggerExecution({ - pieceName, - pieceVersion, - triggerName, - input, - projectId: constants.projectId, - apiUrl: constants.internalApiUrl, - engineToken: constants.engineToken, - devPieces: constants.devPieces, - propertySettings, - stepNames: constants.stepNames, - }) - const isOldVersionOrNotSupported = isNil(pieceTrigger.onStart) - if (isOldVersionOrNotSupported) { - return - } - const context = { - store: createContextStore({ - apiUrl: constants.internalApiUrl, - prefix: '', - flowId: constants.flowId, - engineToken: constants.engineToken, - }), - auth: processedInput[AUTHENTICATION_PROPERTY_NAME], - propsValue: processedInput, - payload, - run: { - id: constants.flowRunId, - }, - step: { - name: triggerName, - }, - project: { - id: constants.projectId, - externalId: constants.externalProjectId, - }, - connections: utils.createConnectionManager({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - target: 'triggers', - contextVersion: piece.getContextInfo?.().version, - pieceName, - }), - } - await pieceTrigger.onStart(context) - }, - - async executeTrigger({ params, constants }: ExecuteTriggerParams): Promise> { - const { pieceName, pieceVersion, triggerName, input, propertySettings } = (params.flowVersion.trigger as PieceTrigger).settings - - if (isNil(triggerName)) { - throw new EngineGenericError('TriggerNameNotSetError', 'Trigger name is not set') - } - - const { piece, pieceTrigger, processedInput } = await prepareTriggerExecution({ - pieceName, - pieceVersion, - triggerName, - input, - projectId: params.projectId, - apiUrl: constants.internalApiUrl, - engineToken: params.engineToken, - devPieces: constants.devPieces, - propertySettings, - stepNames: constants.stepNames, - }) - - const appListeners: Listener[] = [] - const prefix = params.test ? 'test' : '' - let scheduleOptions: ScheduleOptions | undefined = undefined - const context = { - store: createContextStore({ - apiUrl: constants.internalApiUrl, - prefix, - flowId: params.flowVersion.flowId, - engineToken: params.engineToken, - }), - step: { - name: triggerName, - }, - app: { - createListeners({ events, identifierKey, identifierValue }: Listener): void { - appListeners.push({ events, identifierValue, identifierKey }) - }, - }, - setSchedule(request: SetScheduleRequest) { - if ('intervalMs' in request) { - const parsed = ScheduleOptions.safeParse({ - type: TriggerSourceScheduleType.INTERVAL, - intervalMs: request.intervalMs, - }) - if (!parsed.success) { - throw new InvalidScheduleIntervalError(request.intervalMs) - } - scheduleOptions = parsed.data - return - } - if (!isValidCron(request.cronExpression)) { - throw new InvalidCronExpressionError(request.cronExpression) - } - scheduleOptions = { - type: TriggerSourceScheduleType.CRON_EXPRESSION, - cronExpression: request.cronExpression, - timezone: request.timezone ?? 'UTC', - } - }, - flows: createFlowsContext({ - engineToken: params.engineToken, - internalApiUrl: constants.internalApiUrl, - flowId: params.flowVersion.flowId, - flowVersionId: params.flowVersion.id, - }), - webhookUrl: params.webhookUrl, - isRepublish: params.isRepublish, - auth: processedInput[AUTHENTICATION_PROPERTY_NAME], - propsValue: processedInput, - payload: params.triggerPayload ?? {}, - project: { - id: params.projectId, - externalId: constants.externalProjectId, - }, - server: { - token: params.engineToken, - apiUrl: constants.internalApiUrl, - publicUrl: params.publicApiUrl, - }, - connections: utils.createConnectionManager({ - apiUrl: constants.internalApiUrl, - projectId: constants.projectId, - engineToken: constants.engineToken, - target: 'triggers', - contextVersion: piece.getContextInfo?.().version, - pieceName, - }), - } - switch (params.hookType) { - case TriggerHookType.ON_DISABLE: { - await pieceTrigger.onDisable(context) - return {} - } - case TriggerHookType.ON_ENABLE: { - await pieceTrigger.onEnable(context) - return { - listeners: appListeners, - scheduleOptions: pieceTrigger.type === TriggerStrategy.POLLING ? scheduleOptions : undefined, - } - } - case TriggerHookType.RENEW: { - assertEqual(pieceTrigger.type, TriggerStrategy.WEBHOOK, 'triggerType', 'WEBHOOK') - await pieceTrigger.onRenew(context) - return {} - } - case TriggerHookType.HANDSHAKE: { - const { data: handshakeResponse, error: handshakeResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.onHandshake(context)) - - if (handshakeResponseError) { - throw handshakeResponseError - } - return { - response: handshakeResponse, - } - } - case TriggerHookType.TEST: { - const { data: testResponse, error: testResponseError } = await utils.tryCatchAndThrowOnEngineError(() => pieceTrigger.test({ - ...context, - files: createFileUploader({ - apiUrl: constants.internalApiUrl, - engineToken: params.engineToken!, - }), - })) - - if (testResponseError) { - throw testResponseError - } - return { - output: testResponse, - } - } - case TriggerHookType.RUN: { - if (pieceTrigger.type === TriggerStrategy.APP_WEBHOOK) { - - const { data: verified, error: verifiedError } = await utils.tryCatchAndThrowOnEngineError(async () => { - if (!params.appWebhookUrl) { - throw new EngineGenericError('AppWebhookUrlNotAvailableError', `App webhook url is not available for piece name ${pieceName}`) - } - if (!params.webhookSecret) { - throw new EngineGenericError('WebhookSecretNotAvailableError', `Webhook secret is not available for piece name ${pieceName}`) - } - - return piece.events?.verify({ - appWebhookUrl: params.appWebhookUrl, - payload: params.triggerPayload as EventPayload, - webhookSecret: params.webhookSecret, - }) - }) - - if (verifiedError) { - throw verifiedError - } - if (isNil(verified)) { - throw new Error('Webhook is not verified') - } - } - - const { data: triggerRunResult, error: triggerRunError } = await utils.tryCatchAndThrowOnEngineError(async () => { - const items = await pieceTrigger.run({ - ...context, - files: createFileUploader({ - apiUrl: constants.internalApiUrl, - engineToken: params.engineToken!, - }), - }) - return { - output: items, - } - }) - - if (triggerRunError) { - throw triggerRunError - } - return triggerRunResult - } - } - }, -} - -type ExecuteTriggerParams = { - params: ResolvedExecuteTriggerOperation - constants: EngineConstants -} - -async function prepareTriggerExecution({ pieceName, pieceVersion, triggerName, input, propertySettings, projectId, apiUrl, engineToken, devPieces, stepNames }: PrepareTriggerExecutionParams) { - const { piece, pieceTrigger } = await pieceLoader.getPieceAndTriggerOrThrow({ - pieceName, - pieceVersion, - triggerName, - devPieces, - }) - - const { resolvedInput } = await createPropsResolver({ - apiUrl, - projectId, - engineToken, - contextVersion: piece.getContextInfo?.().version, - stepNames, - pieceName, - }).resolve>({ - unresolvedInput: input, - executionState: FlowExecutorContext.empty(), - }) - - const { processedInput, errors } = await propsProcessor.applyProcessorsAndValidators(resolvedInput, pieceTrigger.props, piece.auth, pieceTrigger.requireAuth, propertySettings) - - if (Object.keys(errors).length > 0) { - throw new Error(JSON.stringify(errors, null, 2)) - } - - return { piece, pieceTrigger, processedInput } -} - -type PrepareTriggerExecutionParams = { - pieceName: string - pieceVersion: string - triggerName: string - input: unknown - propertySettings: Record - projectId: string - apiUrl: string - engineToken: string - devPieces: string[] - stepNames: string[] -} diff --git a/packages/server/engine/src/lib/operations/auth-refresh.operation.ts b/packages/server/engine/src/lib/operations/auth-refresh.operation.ts index fc6919767b34..d780e54e19d2 100644 --- a/packages/server/engine/src/lib/operations/auth-refresh.operation.ts +++ b/packages/server/engine/src/lib/operations/auth-refresh.operation.ts @@ -1,21 +1,36 @@ +import { isObject } from '@activepieces/core-utils' +import { PropertyType } from '@activepieces/pieces-framework' import { + AppConnectionType, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthOperation, ExecuteRefreshTokenAuthResponse, } from '@activepieces/shared' -import { EngineConstants } from '../handler/context/engine-constants' -import { pieceHelper } from '../helper/piece-helper' +import { pieceAuth } from '../core/piece/piece-auth' export const authRefreshOperation = { execute: async (operation: ExecuteRefreshTokenAuthOperation): Promise> => { - const output = await pieceHelper.executeRefreshTokenAuth({ - params: operation, - devPieces: EngineConstants.DEV_PIECES, - }) return { status: EngineResponseStatus.OK, - response: output, + response: await refreshAuth(operation), } }, } + +async function refreshAuth(operation: ExecuteRefreshTokenAuthOperation): Promise { + if (operation.auth.type !== AppConnectionType.CUSTOM_AUTH) { + return { skipped: true } + } + const call = await pieceAuth.callMethod({ operation, authValueType: operation.auth.type, methodPath: ['refresh', 'generate'] }) + if (!call.called || call.property.type !== PropertyType.CUSTOM_AUTH || !isObject(call.result) || typeof call.result.access_token !== 'string') { + return { skipped: true } + } + return { + skipped: false, + access_token: call.result.access_token, + expires_in: typeof call.result.expires_in === 'number' ? call.result.expires_in : call.property.refresh?.defaultExpiresIn ?? DEFAULT_REFRESH_EXPIRES_IN_SECONDS, + } +} + +const DEFAULT_REFRESH_EXPIRES_IN_SECONDS = 3300 diff --git a/packages/server/engine/src/lib/operations/auth-validation.operation.ts b/packages/server/engine/src/lib/operations/auth-validation.operation.ts index 8025ba237380..9c338a86d1af 100644 --- a/packages/server/engine/src/lib/operations/auth-validation.operation.ts +++ b/packages/server/engine/src/lib/operations/auth-validation.operation.ts @@ -1,23 +1,36 @@ +import { isObject } from '@activepieces/core-utils' import { EngineResponse, EngineResponseStatus, ExecuteValidateAuthOperation, ExecuteValidateAuthResponse, } from '@activepieces/shared' -import { EngineConstants } from '../handler/context/engine-constants' -import { pieceHelper } from '../helper/piece-helper' +import { pieceAuth } from '../core/piece/piece-auth' export const authValidationOperation = { execute: async (operation: ExecuteValidateAuthOperation): Promise> => { - const input = operation as ExecuteValidateAuthOperation - const output = await pieceHelper.executeValidateAuth({ - params: input, - devPieces: EngineConstants.DEV_PIECES, - }) - + const call = await pieceAuth.callMethod({ operation, authValueType: operation.auth.type, methodPath: ['validate'] }) + if (!call.called) { + return { + status: EngineResponseStatus.OK, + response: call.mismatch + ? { valid: false, error: `Connection value type does not match piece auth type: ${call.property?.type} !== ${operation.auth.type}` } + : { valid: true }, + } + } return { status: EngineResponseStatus.OK, - response: output, + response: toValidateAuthResponse(call.result), } }, -} \ No newline at end of file +} + +function toValidateAuthResponse(value: unknown): ExecuteValidateAuthResponse { + if (!isObject(value)) { + return { valid: false, error: 'Connection validation returned an unexpected result' } + } + if (value.valid === true) { + return { valid: true } + } + return { valid: false, error: typeof value.error === 'string' ? value.error : 'Connection validation failed' } +} diff --git a/packages/server/engine/src/lib/operations/flow.operation.ts b/packages/server/engine/src/lib/operations/flow.operation.ts index b9f04e4c2255..73c01e0ac217 100644 --- a/packages/server/engine/src/lib/operations/flow.operation.ts +++ b/packages/server/engine/src/lib/operations/flow.operation.ts @@ -1,12 +1,12 @@ import { isNil, tryCatch } from '@activepieces/core-utils' import { EngineGenericError, EngineResponse, EngineResponseStatus, ExecuteFlowOperation, ExecuteTriggerResponse, ExecutionError, ExecutionErrorType, ExecutionState, ExecutionType, FlowActionType, FlowRunStatus, flowStructureUtil, GenericStepOutput, LoopStepOutput, ResumePayload, ResumeReason, StepOutput, StepOutputStatus, TriggerHookType, TriggerPayload } from '@activepieces/shared' import { engineFileApi } from '../api/engine-file-api' +import { triggerRunner } from '../core/piece/trigger-runner' import { EngineConstants, ResolvedBeginExecuteFlowOperation, ResolvedExecuteFlowOperation } from '../handler/context/engine-constants' import { FlowExecutorContext } from '../handler/context/flow-execution-context' import { testExecutionContext } from '../handler/context/test-execution-context' import { flowExecutor } from '../handler/flow-executor' import { flowRunProgressReporter } from '../helper/flow-run-progress-reporter' -import { triggerHelper } from '../helper/trigger-helper' import { utils } from '../utils' import { resolveJobPayload } from './utils/resolve-job-payload' @@ -183,7 +183,7 @@ async function runOrReturnPayload(input: ResolvedBeginExecuteFlowOperation, cons if (!input.executeTrigger) { return input.triggerPayload as TriggerPayload } - const newPayload = await triggerHelper.executeTrigger({ + const newPayload = await triggerRunner.executeTrigger({ params: { ...input, hookType: TriggerHookType.RUN, diff --git a/packages/server/engine/src/lib/operations/piece-metadata.operation.ts b/packages/server/engine/src/lib/operations/piece-metadata.operation.ts index d47fcafc37d2..1262acc01f2b 100644 --- a/packages/server/engine/src/lib/operations/piece-metadata.operation.ts +++ b/packages/server/engine/src/lib/operations/piece-metadata.operation.ts @@ -1,23 +1,32 @@ -import { PieceMetadata } from '@activepieces/pieces-framework' +import path from 'path' +import { PieceMetadata, pieceTranslation } from '@activepieces/pieces-framework' import { EngineResponse, EngineResponseStatus, ExecuteExtractPieceMetadataOperation, } from '@activepieces/shared' +import { piecePath } from '../core/piece/piece-path' +import { pieceRunner } from '../core/piece/piece-runner' import { EngineConstants } from '../handler/context/engine-constants' -import { pieceHelper } from '../helper/piece-helper' - export const pieceMetadataOperation = { extract: async (operation: ExecuteExtractPieceMetadataOperation): Promise> => { - const input = operation as ExecuteExtractPieceMetadataOperation - const output = await pieceHelper.extractPieceMetadata({ - params: input, + const piece = { + pieceName: operation.pieceName, + pieceVersion: operation.pieceVersion, devPieces: EngineConstants.DEV_PIECES, - }) + } + const { metadata } = await pieceRunner.describe(piece) + const entryPath = await piecePath.resolve(piece) + const i18n = await pieceTranslation.initializeI18n(path.dirname(path.dirname(entryPath))) return { status: EngineResponseStatus.OK, - response: output, + response: { + ...metadata, + name: operation.pieceName, + version: operation.pieceVersion, + i18n, + }, } }, -} \ No newline at end of file +} diff --git a/packages/server/engine/src/lib/operations/property.operation.ts b/packages/server/engine/src/lib/operations/property.operation.ts index 82027a0f5be5..e4db1b810c7e 100644 --- a/packages/server/engine/src/lib/operations/property.operation.ts +++ b/packages/server/engine/src/lib/operations/property.operation.ts @@ -1,22 +1,138 @@ -import { ExecutePropsResult, PropertyType } from '@activepieces/pieces-framework' +import { isNil, isObject } from '@activepieces/core-utils' +import { DropdownState, ExecutePropsResult, InputPropertyMap, PiecePropertyMap, PropertyType, StaticPropsValue } from '@activepieces/pieces-framework' import { + EngineGenericError, EngineResponse, EngineResponseStatus, ExecutePropsOptions, } from '@activepieces/shared' -import { pieceHelper } from '../helper/piece-helper' - +import * as z from 'zod/mini' +import { PieceDescription } from '../core/piece/piece-protocol' +import { pieceRunner } from '../core/piece/piece-runner' +import { EngineConstants } from '../handler/context/engine-constants' +import { testExecutionContext } from '../handler/context/test-execution-context' +import { buildRuntime } from '../handler/piece-executor' +import { dynamicPropKeys } from '../helper/dynamic-prop-keys' +import { utils } from '../utils' +import { createPropsResolver } from '../variables/props-resolver' export const propertyOperation = { - execute: async (operation: ExecutePropsOptions): Promise>> => { - const output = await pieceHelper.executeProps({ - ...operation, - pieceName: operation.piece.pieceName, - pieceVersion: operation.piece.pieceVersion, - }) + execute: async (operation: ExecutePropsOptions): Promise>> => { return { status: EngineResponseStatus.OK, - response: output, + response: await executeProps(operation), } }, -} \ No newline at end of file +} + +async function executeProps(operation: ExecutePropsOptions): Promise> { + const constants = EngineConstants.fromExecutePropertyInput({ + ...operation, + pieceName: operation.piece.pieceName, + pieceVersion: operation.piece.pieceVersion, + }) + const piece = { + pieceName: operation.piece.pieceName, + pieceVersion: operation.piece.pieceVersion, + devPieces: EngineConstants.DEV_PIECES, + } + const description = await pieceRunner.describe(piece) + const { propertyType, path } = resolvePropertyPath({ description, operation }) + + const { data: result, error } = await utils.tryCatchAndThrowOnEngineError(async () => { + const executionState = await testExecutionContext.stateFromFlowVersion({ + apiUrl: operation.internalApiUrl, + flowVersion: operation.flowVersion, + projectId: operation.projectId, + engineToken: operation.engineToken, + sampleData: operation.sampleData, + engineConstants: constants, + }) + const contextVersion = description.metadata.contextInfo?.version + const { resolvedInput } = await createPropsResolver({ + apiUrl: constants.internalApiUrl, + projectId: constants.projectId, + engineToken: constants.engineToken, + contextVersion, + stepNames: constants.stepNames, + pieceName: piece.pieceName, + }).resolve>({ + unresolvedInput: operation.input, + executionState, + }) + const { result } = await pieceRunner.call({ + piece, + path, + context: { + kind: 'props', + runtime: buildRuntime({ constants, pieceName: piece.pieceName, contextVersion }), + stepName: operation.actionOrTriggerName, + resolvedInput, + searchValue: operation.searchValue, + }, + }) + return result + + }) + + if (error) { + console.error(error) + return { + type: propertyType, + options: { + disabled: true, + options: [], + placeholder: 'Throws an error, reconnect or refresh the page', + }, + } + } + return toPropsResult({ propertyType, result }) +} + +function resolvePropertyPath({ description, operation }: ResolvePropertyPathParams): { propertyType: ExecutablePropertyType, path: string[] } { + const { actionOrTriggerName, propertyName } = operation + const root = isNil(description.metadata.actions[actionOrTriggerName]) ? 'triggers' : 'actions' + const step = description.metadata[root][actionOrTriggerName] + const property = step?.props[propertyName] + if (isNil(property)) { + throw new EngineGenericError('PropertyNotFoundError', `Property not found: ${actionOrTriggerName}.${propertyName}`) + } + if (property.type !== PropertyType.DROPDOWN && property.type !== PropertyType.MULTI_SELECT_DROPDOWN && property.type !== PropertyType.DYNAMIC) { + throw new EngineGenericError('PropertyTypeNotExecutableError', `Property type is not executable: ${property.type} for ${property.displayName}`) + } + return { + propertyType: property.type, + path: [root, actionOrTriggerName, 'props', propertyName, property.type === PropertyType.DYNAMIC ? 'props' : 'options'], + } +} + +function toPropsResult({ propertyType, result }: { propertyType: ExecutablePropertyType, result: unknown }): ExecutePropsResult { + if (propertyType === PropertyType.DYNAMIC) { + return { + type: propertyType, + options: dynamicPropKeys.escapePropsKeys(toInputPropertyMap(result)), + } + } + return { + type: propertyType, + options: toDropdownState(result), + } +} + +function toInputPropertyMap(result: unknown): InputPropertyMap { + return DynamicProps.safeParse(result).data ?? {} +} + +function toDropdownState(result: unknown): DropdownState { + return DropdownResult.safeParse(result).data ?? { disabled: false, options: [] } +} + +const DynamicProps = z.custom((value) => isObject(value)) +const DropdownResult = z.custom>((value) => isObject(value) && Array.isArray(Reflect.get(value, 'options'))) + +type ExecutablePropertyType = PropertyType.DROPDOWN | PropertyType.MULTI_SELECT_DROPDOWN | PropertyType.DYNAMIC + +type ResolvePropertyPathParams = { + description: PieceDescription + operation: ExecutePropsOptions +} diff --git a/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts b/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts index 96b3f3e839ad..a6aef6194769 100644 --- a/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts +++ b/packages/server/engine/src/lib/operations/resolve-connection-identifier.operation.ts @@ -4,19 +4,15 @@ import { ExecuteResolveConnectionIdentifierOperation, ExecuteResolveConnectionIdentifierResponse, } from '@activepieces/shared' -import { EngineConstants } from '../handler/context/engine-constants' -import { pieceHelper } from '../helper/piece-helper' +import { pieceAuth } from '../core/piece/piece-auth' export const resolveConnectionIdentifierOperation = { execute: async (operation: ExecuteResolveConnectionIdentifierOperation): Promise> => { - const output = await pieceHelper.executeResolveConnectionIdentifier({ - params: operation, - devPieces: EngineConstants.DEV_PIECES, - }) - + const call = await pieceAuth.callMethod({ operation, authValueType: operation.connectionType, methodPath: ['getConnectionIdentifier'] }) + const identifier = call.called ? call.result : undefined return { status: EngineResponseStatus.OK, - response: output, + response: { identifier: typeof identifier === 'string' ? identifier : undefined }, } }, } diff --git a/packages/server/engine/src/lib/operations/trigger-hook.operation.ts b/packages/server/engine/src/lib/operations/trigger-hook.operation.ts index bd51a412448c..90b778ead670 100644 --- a/packages/server/engine/src/lib/operations/trigger-hook.operation.ts +++ b/packages/server/engine/src/lib/operations/trigger-hook.operation.ts @@ -1,8 +1,8 @@ import { inspect } from 'util' import { formatPieceError } from '@activepieces/core-utils' import { EngineResponse, EngineResponseStatus, ExecuteTriggerOperation, ExecuteTriggerResponse, TriggerHookType } from '@activepieces/shared' +import { triggerRunner } from '../core/piece/trigger-runner' import { EngineConstants, ResolvedExecuteTriggerOperation } from '../handler/context/engine-constants' -import { triggerHelper } from '../helper/trigger-helper' import { utils } from '../utils' import { resolveJobPayload } from './utils/resolve-job-payload' @@ -18,7 +18,7 @@ export const triggerHookOperation = { }), } const { data: output, error } = await utils.tryCatchAndThrowOnEngineError(() => - triggerHelper.executeTrigger({ + triggerRunner.executeTrigger({ params: input, constants: EngineConstants.fromExecuteTriggerInput(input), }), diff --git a/packages/server/engine/src/piece-child.ts b/packages/server/engine/src/piece-child.ts new file mode 100644 index 000000000000..8df5d36b2fcc --- /dev/null +++ b/packages/server/engine/src/piece-child.ts @@ -0,0 +1,3 @@ +import { pieceChild } from './lib/core/piece/piece-child' + +pieceChild.listen() diff --git a/packages/server/engine/test/core/piece/piece-memory.test.ts b/packages/server/engine/test/core/piece/piece-memory.test.ts new file mode 100644 index 000000000000..279379df8818 --- /dev/null +++ b/packages/server/engine/test/core/piece/piece-memory.test.ts @@ -0,0 +1,26 @@ +import { ExecutionError, ExecutionErrorType } from '@activepieces/shared' +import { toExitError } from '../../../src/lib/core/piece/piece-runner' + +const heapMessage = 'FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory' + +describe('piece child exit classification', () => { + it.each([ + ['the V8 heap message', { code: 1, signal: null, output: heapMessage }], + ['an abort exit code', { code: 134, signal: null, output: '' }], + ['SIGABRT', { code: null, signal: 'SIGABRT' as const, output: '' }], + ['a kernel OOM kill', { code: null, signal: 'SIGKILL' as const, output: '' }], + ])('reports %s as a user-level memory failure', (_name, exit) => { + const error = toExitError(exit) + + expect(error).toBeInstanceOf(ExecutionError) + expect(error).toMatchObject({ name: 'PieceMemoryLimitError', type: ExecutionErrorType.USER }) + expect(JSON.parse(error.message).message).toBe('The piece ran out of memory') + }) + + it('reports any other abnormal exit as an engine error carrying the child output', () => { + const error = toExitError({ code: 7, signal: null, output: 'some stack trace' }) + + expect(error).toMatchObject({ name: 'PieceProcessExitedError', type: ExecutionErrorType.ENGINE }) + expect(error.message).toContain('some stack trace') + }) +}) diff --git a/packages/server/engine/test/core/piece/piece-protocol.test.ts b/packages/server/engine/test/core/piece/piece-protocol.test.ts new file mode 100644 index 000000000000..c4a507825625 --- /dev/null +++ b/packages/server/engine/test/core/piece/piece-protocol.test.ts @@ -0,0 +1,48 @@ +import { ExecutionError, ExecutionErrorType } from '@activepieces/shared' +import { pieceProtocol } from '../../../src/lib/core/piece/piece-protocol' + +describe('piece protocol', () => { + it('keeps the execution error type across the boundary', () => { + for (const type of [ExecutionErrorType.ENGINE, ExecutionErrorType.USER]) { + const restored = pieceProtocol.deserializeError(pieceProtocol.serializeError(new ExecutionError('BoomError', 'boom', type))) + + expect(restored).toBeInstanceOf(ExecutionError) + expect(restored).toMatchObject({ name: 'BoomError', message: 'boom', type }) + } + }) + + it('keeps http details and the constructor name of a plain piece error', () => { + class HttpError extends Error { + constructor(readonly status: number) { + super('request failed') + } + } + + const restored = pieceProtocol.deserializeError(pieceProtocol.serializeError(new HttpError(404))) + + expect(restored).not.toBeInstanceOf(ExecutionError) + expect(restored).toMatchObject({ name: 'HttpError', message: 'request failed', status: 404 }) + }) + + it('falls back to a message for a thrown non-error', () => { + expect(pieceProtocol.serializeError('just a string')).toEqual({ message: 'just a string' }) + }) + + it('drops functions and promises from a result but keeps buffers and dates', () => { + const date = new Date('2020-01-01T00:00:00.000Z') + + const transferable = pieceProtocol.toTransferable({ + keep: Buffer.from('bytes'), + when: date, + nested: { fn: () => undefined, pending: Promise.resolve(1), value: 2 }, + list: [1, () => undefined, 'three'], + }) + + expect(transferable).toEqual({ + keep: Buffer.from('bytes'), + when: date, + nested: { value: 2 }, + list: [1, undefined, 'three'], + }) + }) +}) diff --git a/packages/server/engine/test/handler/flow-log-size.test.ts b/packages/server/engine/test/handler/flow-log-size.test.ts index 49065fb938c3..044b189201f9 100644 --- a/packages/server/engine/test/handler/flow-log-size.test.ts +++ b/packages/server/engine/test/handler/flow-log-size.test.ts @@ -13,8 +13,8 @@ vi.mock('../../src/lib/helper/flow-run-progress-reporter', () => ({ }, })) -vi.mock('../../src/lib/helper/trigger-helper', () => ({ - triggerHelper: { +vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ + triggerRunner: { executeOnStart: vi.fn().mockResolvedValue(undefined), }, })) diff --git a/packages/server/engine/test/handler/flow-waitpoint-response.test.ts b/packages/server/engine/test/handler/flow-waitpoint-response.test.ts index ceee9c5d7555..9684c7a0a50c 100644 --- a/packages/server/engine/test/handler/flow-waitpoint-response.test.ts +++ b/packages/server/engine/test/handler/flow-waitpoint-response.test.ts @@ -2,17 +2,13 @@ import { FlowRunStatus } from '@activepieces/shared' import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { flowExecutor } from '../../src/lib/handler/flow-executor' +import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' import { buildPieceAction, generateMockEngineConstants } from './test-helper' const { mockSendFlowResponse } = vi.hoisted(() => ({ mockSendFlowResponse: vi.fn().mockResolvedValue(undefined), })) -vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ - waitpointClient: { - create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), - }, -})) vi.mock('../../src/lib/api/engine-run-api', () => ({ engineRunApi: { @@ -21,6 +17,17 @@ vi.mock('../../src/lib/api/engine-run-api', () => ({ })) describe('flow waitpoint response propagation', () => { + let engineApi: EngineApiStub + + beforeEach(async () => { + engineApi = await startEngineApiStub({ + 'POST /v1/waitpoints': { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, + }) + }) + + afterEach(async () => { + await engineApi.close() + }) beforeEach(() => { vi.clearAllMocks() @@ -48,6 +55,7 @@ describe('flow waitpoint response propagation', () => { action, executionState: FlowExecutorContext.empty(), constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, triggerPieceName: '@activepieces/piece-webhook', workerHandlerId: 'test-handler-id', httpRequestId: 'test-request-id', @@ -71,6 +79,9 @@ describe('flow waitpoint response propagation', () => { }, }, }) + const sentHeaders = mockSendFlowResponse.mock.calls[0][0].request.runResponse.headers + expect(typeof sentHeaders['x-activepieces-resume-webhook-url']).toBe('string') + expect(sentHeaders['x-activepieces-resume-webhook-url']).toMatch(/^https?:\/\//) }) it('should not call sendFlowResponse when triggerPieceName does not match', async () => { @@ -92,6 +103,7 @@ describe('flow waitpoint response propagation', () => { action, executionState: FlowExecutorContext.empty(), constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, triggerPieceName: 'some-other-piece', workerHandlerId: 'test-handler-id', httpRequestId: 'test-request-id', diff --git a/packages/server/engine/test/handler/flow-with-delay.test.ts b/packages/server/engine/test/handler/flow-with-delay.test.ts index b219cf2f424e..d299431ab776 100644 --- a/packages/server/engine/test/handler/flow-with-delay.test.ts +++ b/packages/server/engine/test/handler/flow-with-delay.test.ts @@ -1,20 +1,22 @@ import { FlowRunStatus } from '@activepieces/shared' -import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { flowExecutor } from '../../src/lib/handler/flow-executor' -import { waitpointClient } from '../../src/lib/piece-context/waitpoint-client' +import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' import { buildCodeAction, buildPieceAction, generateMockEngineConstants } from './test-helper' -vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ - waitpointClient: { - create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), - }, -})) +const WAITPOINT_PATH = '/v1/waitpoints' describe('flow with delay', () => { + let engineApi: EngineApiStub - beforeEach(() => { - vi.clearAllMocks() + beforeEach(async () => { + engineApi = await startEngineApiStub({ + [`POST ${WAITPOINT_PATH}`]: { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, + }) + }) + + afterEach(async () => { + await engineApi.close() }) it('delay-for pauses flow and calls waitpointClient.create with DELAY type', async () => { @@ -35,13 +37,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayForFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(result.verdict).toEqual({ status: FlowRunStatus.PAUSED, }) - expect(waitpointClient.create).toHaveBeenCalledWith( + expect(engineApi.requestsFor(WAITPOINT_PATH)[0].body).toEqual( expect.objectContaining({ type: 'DELAY', resumeDateTime: expect.any(String), @@ -67,7 +69,7 @@ describe('flow with delay', () => { const pauseResult = await flowExecutor.execute({ action: delayForFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) const resumeResult = await flowExecutor.execute({ @@ -76,6 +78,7 @@ describe('flow with delay', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, resumePayload: { queryParams: {}, body: {}, @@ -106,13 +109,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: shortDelayFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(result.verdict).toEqual({ status: FlowRunStatus.RUNNING, }) - expect(waitpointClient.create).not.toHaveBeenCalled() + expect(engineApi.requestsFor(WAITPOINT_PATH)).toHaveLength(0) }) it('delay-until pauses flow for future dates', async () => { @@ -133,13 +136,13 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayUntilFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(result.verdict).toEqual({ status: FlowRunStatus.PAUSED, }) - expect(waitpointClient.create).toHaveBeenCalledWith( + expect(engineApi.requestsFor(WAITPOINT_PATH)[0].body).toEqual( expect.objectContaining({ type: 'DELAY', resumeDateTime: expect.any(String), @@ -161,12 +164,12 @@ describe('flow with delay', () => { const result = await flowExecutor.execute({ action: delayUntilFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(result.verdict).toEqual({ status: FlowRunStatus.RUNNING, }) - expect(waitpointClient.create).not.toHaveBeenCalled() + expect(engineApi.requestsFor(WAITPOINT_PATH)).toHaveLength(0) }) }) diff --git a/packages/server/engine/test/handler/flow-with-pause.test.ts b/packages/server/engine/test/handler/flow-with-pause.test.ts index effea6b03974..a047ad7be6ea 100644 --- a/packages/server/engine/test/handler/flow-with-pause.test.ts +++ b/packages/server/engine/test/handler/flow-with-pause.test.ts @@ -3,13 +3,9 @@ import { vi } from 'vitest' import { FlowExecutorContext } from '../../src/lib/handler/context/flow-execution-context' import { StepExecutionPath } from '../../src/lib/handler/context/step-execution-path' import { flowExecutor } from '../../src/lib/handler/flow-executor' +import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' import { buildCodeAction, buildPieceAction, buildRouterWithOneCondition, buildSimpleLoopAction, generateMockEngineConstants } from './test-helper' -vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ - waitpointClient: { - create: vi.fn().mockResolvedValue({ id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }), - }, -})) const simplePauseFlow = buildPieceAction({ @@ -65,12 +61,23 @@ const pauseFlowWithLoopAndBranch = buildSimpleLoopAction({ }) describe('flow with pause', () => { + let engineApi: EngineApiStub + + beforeEach(async () => { + engineApi = await startEngineApiStub({ + 'POST /v1/waitpoints': { id: 'mock-waitpoint-id', resumeUrl: 'http://localhost/resume' }, + }) + }) + + afterEach(async () => { + await engineApi.close() + }) it('should pause and resume successfully with loops and branch', async () => { const pauseResult = await flowExecutor.execute({ action: pauseFlowWithLoopAndBranch, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants({ stepNames: ['loop'] }), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url, stepNames: ['loop'] }), }) expect(pauseResult.verdict).toEqual({ status: FlowRunStatus.PAUSED, @@ -90,6 +97,7 @@ describe('flow with pause', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, stepNames: ['loop'], resumePayload: { queryParams: { @@ -119,12 +127,13 @@ describe('flow with pause', () => { const pauseResult1 = await flowExecutor.execute({ action: flawWithTwoPause, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) const resumeResult1 = await flowExecutor.execute({ action: flawWithTwoPause, executionState: pauseResult1, constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -143,6 +152,7 @@ describe('flow with pause', () => { status: FlowRunStatus.RUNNING, }), constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -163,7 +173,7 @@ describe('flow with pause', () => { const pauseResult = await flowExecutor.execute({ action: simplePauseFlow, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(pauseResult.verdict).toStrictEqual({ status: FlowRunStatus.PAUSED, @@ -175,6 +185,7 @@ describe('flow with pause', () => { action: simplePauseFlow, executionState: pauseResult, constants: generateMockEngineConstants({ + internalApiUrl: engineApi.url, resumePayload: { queryParams: { action: 'approve', @@ -237,7 +248,7 @@ describe('flow with pause', () => { const result = await flowExecutor.execute({ action: routerWithTwoPauseActions, executionState: FlowExecutorContext.empty(), - constants: generateMockEngineConstants(), + constants: generateMockEngineConstants({ internalApiUrl: engineApi.url }), }) expect(result.verdict).toStrictEqual({ diff --git a/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts b/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts index 3a9282097f32..8ff996625f58 100644 --- a/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts +++ b/packages/server/engine/test/helper/flow-run-progress-reporter.test.ts @@ -170,7 +170,7 @@ describe('flow-run-progress-reporter slicing in single-step test mode', () => { logsFileId: 'logs-1', }) - const outputContext = flowRunProgressReporter.createOutputContext({ engineConstants }) + const outputContext = flowRunProgressReporter.createOutputContext(engineConstants) const big = { big: 'x'.repeat(40_000) } await outputContext.update({ data: big }) @@ -195,7 +195,7 @@ describe('flow-run-progress-reporter slicing in single-step test mode', () => { }) updateStepProgressMock.mockRejectedValueOnce(new Error('Failed to POST step-progress: 400 Bad Request')) - const outputContext = flowRunProgressReporter.createOutputContext({ engineConstants }) + const outputContext = flowRunProgressReporter.createOutputContext(engineConstants) // Must resolve, not reject — a streaming failure must not fail the run. await expect(outputContext.update({ data: { partial: true } })).resolves.toBeUndefined() diff --git a/packages/server/engine/test/helpers/engine-api-stub.ts b/packages/server/engine/test/helpers/engine-api-stub.ts new file mode 100644 index 000000000000..0727630331f3 --- /dev/null +++ b/packages/server/engine/test/helpers/engine-api-stub.ts @@ -0,0 +1,65 @@ +import { createServer, IncomingMessage, ServerResponse } from 'node:http' +import { AddressInfo } from 'node:net' + +export async function startEngineApiStub(routes: Routes = {}): Promise { + const requests: RecordedRequest[] = [] + + const server = createServer((req, res) => void handle({ req, res, routes, requests })) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + + return { + url: `http://127.0.0.1:${port}/`, + requests, + requestsFor: (path: string) => requests.filter((request) => request.path === path), + close: async () => new Promise((resolve) => server.close(() => resolve())), + } +} + +async function handle({ req, res, routes, requests }: HandleParams): Promise { + const path = (req.url ?? '').split('?')[0] + const body = await readBody(req) + requests.push({ method: req.method ?? 'GET', path, body }) + + const route = routes[`${req.method} ${path}`] ?? routes[path] + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify(route ?? {})) +} + +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = [] + for await (const chunk of req) { + chunks.push(Buffer.from(chunk)) + } + if (chunks.length === 0) { + return undefined + } + try { + return JSON.parse(Buffer.concat(chunks).toString()) + } + catch { + return Buffer.concat(chunks).toString() + } +} + +type Routes = Record + +type HandleParams = { + req: IncomingMessage + res: ServerResponse + routes: Routes + requests: RecordedRequest[] +} + +export type RecordedRequest = { + method: string + path: string + body: unknown +} + +export type EngineApiStub = { + url: string + requests: RecordedRequest[] + requestsFor: (path: string) => RecordedRequest[] + close: () => Promise +} diff --git a/packages/server/engine/test/operations/flow-operation-invariants.test.ts b/packages/server/engine/test/operations/flow-operation-invariants.test.ts index 4043cedf2f69..be94321c5fca 100644 --- a/packages/server/engine/test/operations/flow-operation-invariants.test.ts +++ b/packages/server/engine/test/operations/flow-operation-invariants.test.ts @@ -31,8 +31,8 @@ vi.mock('../../src/lib/helper/flow-run-progress-reporter', () => ({ const { mockExecuteTrigger } = vi.hoisted(() => ({ mockExecuteTrigger: vi.fn(), })) -vi.mock('../../src/lib/helper/trigger-helper', () => ({ - triggerHelper: { +vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ + triggerRunner: { executeTrigger: mockExecuteTrigger, executeOnStart: vi.fn().mockResolvedValue(undefined), }, @@ -49,16 +49,9 @@ vi.mock('../../src/lib/api/engine-file-api', () => ({ }, })) -const { mockCreateWaitpoint } = vi.hoisted(() => ({ - mockCreateWaitpoint: vi.fn(), -})) -vi.mock('../../src/lib/piece-context/waitpoint-client', () => ({ - waitpointClient: { - create: mockCreateWaitpoint, - }, -})) import { flowOperation } from '../../src/lib/operations/flow.operation' +import { EngineApiStub, startEngineApiStub } from '../helpers/engine-api-stub' function makeFlowVersion(): FlowVersion { return { @@ -89,7 +82,7 @@ function makeBeginOperation(overrides?: Partial): Beg return { projectId: 'proj-1', engineToken: 'test-token', - internalApiUrl: 'http://localhost:3000/', + internalApiUrl: engineApi.url, publicApiUrl: 'http://localhost:4200/api/', timeoutInSeconds: 600, platformId: 'plat-1', @@ -158,7 +151,7 @@ function makeResumeOperation(overrides?: Partial): R return { projectId: 'proj-1', engineToken: 'test-token', - internalApiUrl: 'http://localhost:3000/', + internalApiUrl: engineApi.url, publicApiUrl: 'http://localhost:4200/api/', timeoutInSeconds: 600, platformId: 'plat-1', @@ -177,7 +170,18 @@ function makeResumeOperation(overrides?: Partial): R } } +let engineApi: EngineApiStub + describe('flow operation invariants', () => { + beforeEach(async () => { + engineApi = await startEngineApiStub({ + 'POST /v1/waitpoints': { id: 'wp-new', resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-new' }, + }) + }) + + afterEach(async () => { + await engineApi.close() + }) describe('RESUME execution state hydration', () => { it('throws EngineGenericError when RESUME has empty execution state in logs file', async () => { mockDownload.mockReset() @@ -268,11 +272,6 @@ describe('flow operation invariants', () => { // (waitpoint path). With the fix, step_1 stays FAILED in the restored state, // `isCompleted` short-circuits piece-executor, and no new waitpoint is created. mockDownload.mockReset() - mockCreateWaitpoint.mockReset() - mockCreateWaitpoint.mockResolvedValue({ - id: 'wp-new', - resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-new', - }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -313,7 +312,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(mockCreateWaitpoint).not.toHaveBeenCalled() + expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(0) }) it('drops FAILED steps on a retry resume (resumeReason=RETRY — FlowRetryStrategy.FROM_FAILED_STEP)', async () => { @@ -322,11 +321,6 @@ describe('flow operation invariants', () => { // engine to replay the failed step. Preserving FAILED on this path would silently turn // retry into a no-op. The discriminator is the explicit `resumeReason` field. mockDownload.mockReset() - mockCreateWaitpoint.mockReset() - mockCreateWaitpoint.mockResolvedValue({ - id: 'wp-retry', - resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-retry', - }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -361,7 +355,7 @@ describe('flow operation invariants', () => { // step_1 (FAILED) was dropped because resumeReason=RETRY → engine replayed it from // BEGIN, which creates a waitpoint via the approval piece. - expect(mockCreateWaitpoint).toHaveBeenCalled() + expect(engineApi.requestsFor('/v1/waitpoints').length).toBeGreaterThan(0) }) it('drops non-terminal statuses (e.g. RUNNING from a mid-step crash) on any resume', async () => { @@ -370,11 +364,6 @@ describe('flow operation invariants', () => { // whether resumePayload is present. Only SUCCEEDED, PAUSED, and FAILED (the last // conditionally) survive restoration. mockDownload.mockReset() - mockCreateWaitpoint.mockReset() - mockCreateWaitpoint.mockResolvedValue({ - id: 'wp-replay', - resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-replay', - }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -414,7 +403,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(mockCreateWaitpoint).toHaveBeenCalledTimes(1) + expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(1) }) it('preserves FAILED steps on a delay-piece waitpoint resume even though resumePayload is null', async () => { @@ -424,11 +413,6 @@ describe('flow operation invariants', () => { // engine would drop FAILED — replaying any `continueOnFailure` step that preceded // the delay. With `resumeReason: WAITPOINT`, FAILED is preserved correctly. mockDownload.mockReset() - mockCreateWaitpoint.mockReset() - mockCreateWaitpoint.mockResolvedValue({ - id: 'wp-delay', - resumeUrl: 'http://localhost:4200/api/v1/flow-runs/run-1/waitpoints/wp-delay', - }) mockDownload.mockResolvedValue( new TextEncoder().encode(JSON.stringify({ @@ -467,7 +451,7 @@ describe('flow operation invariants', () => { await flowOperation.execute(operation) - expect(mockCreateWaitpoint).not.toHaveBeenCalled() + expect(engineApi.requestsFor('/v1/waitpoints')).toHaveLength(0) }) }) @@ -503,7 +487,7 @@ describe('flow operation invariants', () => { } expect(mockDownload).toHaveBeenCalledWith({ - apiUrl: 'http://localhost:3000/', + apiUrl: engineApi.url, engineToken: 'test-token', fileId: 'payload-file-1', }) @@ -633,7 +617,6 @@ describe('flow operation invariants', () => { describe('RESUME payload hydration', () => { it('resolves a ref resumePayload via the engine file client', async () => { mockDownload.mockReset() - mockCreateWaitpoint.mockReset() mockDownload.mockImplementation(({ fileId }: { fileId: string }) => { if (fileId === 'logs-file-1') { return Promise.resolve(new TextEncoder().encode(JSON.stringify({ @@ -665,7 +648,7 @@ describe('flow operation invariants', () => { } expect(mockDownload).toHaveBeenCalledWith({ - apiUrl: 'http://localhost:3000/', + apiUrl: engineApi.url, engineToken: 'test-token', fileId: 'resume-file-1', }) diff --git a/packages/server/engine/test/operations/trigger-hook-operation.test.ts b/packages/server/engine/test/operations/trigger-hook-operation.test.ts index 11ead75f56c1..caec7d824493 100644 --- a/packages/server/engine/test/operations/trigger-hook-operation.test.ts +++ b/packages/server/engine/test/operations/trigger-hook-operation.test.ts @@ -19,8 +19,8 @@ vi.mock('../../src/lib/api/engine-file-api', () => ({ const { mockExecuteTrigger } = vi.hoisted(() => ({ mockExecuteTrigger: vi.fn(), })) -vi.mock('../../src/lib/helper/trigger-helper', () => ({ - triggerHelper: { +vi.mock('../../src/lib/core/piece/trigger-runner', () => ({ + triggerRunner: { executeTrigger: mockExecuteTrigger, }, })) diff --git a/packages/server/engine/vitest.config.ts b/packages/server/engine/vitest.config.ts index 8b3a73c79e8c..fd25c528cff3 100644 --- a/packages/server/engine/vitest.config.ts +++ b/packages/server/engine/vitest.config.ts @@ -1,4 +1,5 @@ import path from 'path' +import { buildSync } from 'esbuild' import { defineConfig } from 'vitest/config' // Change CWD to repo root for compatibility with piece-loader path resolution @@ -10,6 +11,29 @@ process.env.AP_BASE_CODE_DIRECTORY = 'packages/server/engine/test/resources/code process.env.AP_TEST_MODE = 'true' process.env.AP_DEV_PIECES = 'http,data-mapper,approval,webhook,delay' +const alias = { + '@activepieces/shared': path.resolve(__dirname, '../../core/shared/src/index.ts'), + '@activepieces/pieces-framework': path.resolve(__dirname, '../../pieces/framework/src/index.ts'), + '@activepieces/pieces-common': path.resolve(__dirname, '../../pieces/common/src/index.ts'), + '@activepieces/core-formula': path.resolve(__dirname, '../../core/formula/src/index.ts'), + '@activepieces/core-piece-types': path.resolve(__dirname, '../../core/piece-types/src/index.ts'), + '@activepieces/core-utils': path.resolve(__dirname, '../../core/utils/src/index.ts'), + '@activepieces/core-execution': path.resolve(__dirname, '../../core/execution/src/index.ts'), +} + +const pieceChildEntry = path.resolve(__dirname, '../../../dist/packages/engine-test/piece-child.js') +buildSync({ + entryPoints: [path.resolve(__dirname, 'src/piece-child.ts')], + bundle: true, + platform: 'node', + target: 'node20', + outfile: pieceChildEntry, + format: 'cjs', + alias, + external: ['isolated-vm', 'utf-8-validate', 'bufferutil'], +}) +process.env.AP_PIECE_CHILD_ENTRY = pieceChildEntry + export default defineConfig({ // esbuild injects this at bundle time; vitest runs the source directly, so define it here too. // Tests exercise the proxy-included path (the no-proxy bundle's behaviour is the build-flag flip). @@ -23,14 +47,6 @@ export default defineConfig({ include: [path.resolve(__dirname, 'test/**/*.test.ts')], }, resolve: { - alias: { - '@activepieces/shared': path.resolve(__dirname, '../../../packages/core/shared/src/index.ts'), - '@activepieces/pieces-framework': path.resolve(__dirname, '../../../packages/pieces/framework/src/index.ts'), - '@activepieces/pieces-common': path.resolve(__dirname, '../../../packages/pieces/common/src/index.ts'), - '@activepieces/core-formula': path.resolve(__dirname, '../../../packages/core/formula/src/index.ts'), - '@activepieces/core-piece-types': path.resolve(__dirname, '../../../packages/core/piece-types/src/index.ts'), - '@activepieces/core-utils': path.resolve(__dirname, '../../../packages/core/utils/src/index.ts'), - '@activepieces/core-execution': path.resolve(__dirname, '../../../packages/core/execution/src/index.ts'), - }, + alias, }, }) diff --git a/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts b/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts index a788db11812b..975f0f863e6a 100644 --- a/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts +++ b/packages/server/sandbox/src/lib/cache/engine/engine-installer.ts @@ -8,20 +8,19 @@ import { ApEnvironment } from '@activepieces/shared' import { nanoid } from 'nanoid' import { SandboxSettings } from '../../types' -const engineExecutablePath = 'dist/packages/engine/main.js' +const engineDistPath = 'dist/packages/engine' +const engineBundles = ['main.js', 'piece-child.js'] const installedPaths = new Map>() export const engineInstaller = (_log: ApLogger, getSettings: () => SandboxSettings) => ({ async install({ path }: InstallParams): Promise { const isDev = getSettings().ENVIRONMENT === ApEnvironment.DEVELOPMENT - // The egress proxy was removed, so there is a single engine bundle (main.js). - const source = engineExecutablePath const inFlight = installedPaths.get(path) if (!isNil(inFlight) && !isDev) { await inFlight return { cacheHit: true } } - const install = copyEngine({ source, path }) + const install = copyEngine({ path }) installedPaths.set(path, install) const { error } = await tryCatch(() => install) if (error) { @@ -32,9 +31,11 @@ export const engineInstaller = (_log: ApLogger, getSettings: () => SandboxSettin }, }) -async function copyEngine({ source, path }: CopyEngineParams): Promise { - await atomicCopy(source, `${path}/main.js`) - await atomicCopy(`${source}.map`, `${path}/main.js.map`) +async function copyEngine({ path }: CopyEngineParams): Promise { + for (const bundle of engineBundles) { + await atomicCopy(`${engineDistPath}/${bundle}`, `${path}/${bundle}`) + await atomicCopy(`${engineDistPath}/${bundle}.map`, `${path}/${bundle}.map`) + } } async function atomicCopy(src: PathLike, dest: PathLike): Promise { @@ -46,7 +47,6 @@ async function atomicCopy(src: PathLike, dest: PathLike): Promise { } type CopyEngineParams = { - source: string path: string } diff --git a/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts b/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts index 609bab38522f..8711a0b8ceb3 100644 --- a/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts +++ b/packages/server/sandbox/test/lib/cache/engine-installer-identity.test.ts @@ -16,6 +16,8 @@ async function makeSandboxRoot(): Promise { await mkdir(join(root, ENGINE_SOURCE_DIR), { recursive: true }) await writeFile(join(root, ENGINE_SOURCE_DIR, 'main.js'), 'engine-bundle', 'utf8') await writeFile(join(root, ENGINE_SOURCE_DIR, 'main.js.map'), '{}', 'utf8') + await writeFile(join(root, ENGINE_SOURCE_DIR, 'piece-child.js'), 'piece-child-bundle', 'utf8') + await writeFile(join(root, ENGINE_SOURCE_DIR, 'piece-child.js.map'), '{}', 'utf8') const target = join(root, 'cache', 'common') await mkdir(target, { recursive: true }) process.chdir(root) @@ -51,6 +53,7 @@ describe('engineInstaller', () => { expect(second.cacheHit).toBe(true) expect(third.cacheHit).toBe(true) expect(await readFile(join(target, 'main.js'), 'utf8')).toBe('engine-bundle') + expect(await readFile(join(target, 'piece-child.js'), 'utf8')).toBe('piece-child-bundle') }) it('is not invalidated by another container writing the shared cache.json', async () => { From 70be1438cf221a76caf86cd24c5d52a8975f2a46 Mon Sep 17 00:00:00 2001 From: Bartosz Majewski <30874844+majewskibartosz@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:21:02 +0200 Subject: [PATCH 4/6] fix(connections): stop oauth2 connections failing to save when the code has special characters (#14879) --- .../connections/utils/oauth2-utils.ts | 2 +- .../oauth2-authorization-code-decode.test.ts | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packages/web/test/features/connections/utils/oauth2-authorization-code-decode.test.ts diff --git a/packages/web/src/features/connections/utils/oauth2-utils.ts b/packages/web/src/features/connections/utils/oauth2-utils.ts index de0ab02fa042..4fd51b3e27f6 100644 --- a/packages/web/src/features/connections/utils/oauth2-utils.ts +++ b/packages/web/src/features/connections/utils/oauth2-utils.ts @@ -76,7 +76,7 @@ function getCode(redirectUrl: string): Promise { redirectUrl.startsWith(event.origin) && event.data['code'] ) { - resolve(decodeURIComponent(event.data.code)); + resolve(event.data.code); closeOAuth2Popup(); window.removeEventListener('message', handler); } diff --git a/packages/web/test/features/connections/utils/oauth2-authorization-code-decode.test.ts b/packages/web/test/features/connections/utils/oauth2-authorization-code-decode.test.ts new file mode 100644 index 000000000000..54ba66f6a24a --- /dev/null +++ b/packages/web/test/features/connections/utils/oauth2-authorization-code-decode.test.ts @@ -0,0 +1,61 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from 'vitest'; + +import { oauth2Utils } from '@/features/connections/utils/oauth2-utils'; + +const REDIRECT_URL = 'http://localhost/redirect'; + +function codeAsPostedByRedirectPage(issuedCode: string): string { + const redirectSearch = `?code=${encodeURIComponent(issuedCode)}`; + const code = new URLSearchParams(redirectSearch).get('code'); + if (code === null) { + throw new Error('code param missing'); + } + return code; +} + +function dispatchCodeMessage(code: string): void { + window.dispatchEvent( + new MessageEvent('message', { + data: { code }, + origin: 'http://localhost', + }), + ); +} + +async function codeReachingTokenExchange(issuedCode: string): Promise { + const response = oauth2Utils.openOAuth2Popup({ + authorizationUrl: 'https://provider.example/authorize', + redirectUrl: REDIRECT_URL, + }); + dispatchCodeMessage(codeAsPostedByRedirectPage(issuedCode)); + const { code } = await response; + return code; +} + +describe('OAuth2 authorization code decode (GIT-1763)', () => { + it('preserves a code containing a percent-encoded sequence', async () => { + const issuedCode = 'k1%2Fk2'; + expect(await codeReachingTokenExchange(issuedCode)).toBe(issuedCode); + }); + + it('resolves without hanging when the code contains a stray percent sign', async () => { + const issuedCode = 'abc%zzdef'; + + const outcome = await Promise.race([ + codeReachingTokenExchange(issuedCode), + new Promise<'still-pending'>((resolve) => + setTimeout(() => resolve('still-pending'), 100), + ), + ]); + + expect(outcome).toBe(issuedCode); + }); + + it('leaves URL-safe codes unchanged', async () => { + const issuedCode = 'plainSafeCode-123_456.789'; + expect(await codeReachingTokenExchange(issuedCode)).toBe(issuedCode); + }); +}); From c52c65b2915ec469bfa3bfedda49d41f59f2b7f1 Mon Sep 17 00:00:00 2001 From: Bartosz Majewski <30874844+majewskibartosz@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:21:16 +0200 Subject: [PATCH 5/6] fix(flows): entered values no longer reset when changing fields they depend on (#14357) Co-authored-by: louai --- brain/knowledge/flows-execution/flows.md | 1 + brain/knowledge/pieces-engine/pieces.md | 3 + .../dynamic-piece-property.tsx | 59 ++-- .../web/src/features/pieces/api/pieces-api.ts | 15 +- .../dynamic-piece-property.test.tsx | 262 ++++++++++++++++++ .../test/features/pieces/pieces-api.test.ts | 43 +++ 6 files changed, 350 insertions(+), 33 deletions(-) create mode 100644 packages/web/test/app/builder/piece-properties/dynamic-piece-property.test.tsx create mode 100644 packages/web/test/features/pieces/pieces-api.test.ts diff --git a/brain/knowledge/flows-execution/flows.md b/brain/knowledge/flows-execution/flows.md index 2aa988a30886..9b39ac85d103 100644 --- a/brain/knowledge/flows-execution/flows.md +++ b/brain/knowledge/flows-execution/flows.md @@ -19,6 +19,7 @@ Flows are the core automation primitive: a versioned directed graph of trigger + - Sample data is captured per step (input+output) as File entities per flow version. ### Gotchas +- **Step settings autosave from inside the form resolver, on every validating `setValue`.** `step-settings/index.tsx` runs `applyOperation(UPDATE_ACTION/UPDATE_TRIGGER)` in its `resolver` whenever the new values differ from the last saved snapshot — it is *not* gated on `isDirty` or on a submit. Any transient value a component writes with `shouldValidate: true` is therefore persisted immediately, including one it intends to overwrite a moment later from an async response. - **A CODE step's compiled size is its `packageJson`, not its code — bundling *inlines* `node_modules`, it does not exclude it.** This gets assumed backwards a lot: there is no `node_modules` at runtime precisely *because* esbuild inlines every dependency into the step's `index.js`. Measured on cloud, Aug 2026: a step with **1,870 characters** of user source and `{"pdfkit":"0.14.0","aws-sdk":"2.1531.0","uuid":"9.0.1"}` compiles to **24.13 MB**, of which 24.13 MB is `node_modules` — **21.16 MB of it `aws-sdk` alone**. v2 of that SDK resolves its ~200 service clients by dynamic `require`, so esbuild cannot tree-shake it and inlines all of them; `@aws-sdk/client-*` (v3) would be a few hundred KB. Fleet-wide there were 13,918 compiled steps totalling 1.4 GB, **61 of them over 10 MB**, and per *flow* the totals reach **190.7 MB across 40 code steps**. That per-flow number is the one that matters operationally, because `flowBundleStore.publish` holds a flow's entire compiled output in memory at once (three copies — see the OOM gotcha on [[workers]]). To find the offenders: esbuild leaves `// node_modules//…` markers in the output, so you can attribute bytes per package by summing the lines between markers. - **Step output nesting (schema v21+)**: every step output is wrapped as `{ output, error? }`; expressions must use the `['output']` accessor. The v20→v21 migration rewrites existing expressions via `expression-rewriter`. - **Continue on Failure**: CODE/PIECE steps with `continueOnFailure.value: true` carry `onSuccess`/`onFailure` sub-trees under `settings.errorHandlingOptions.continueOnFailureBranches`. diff --git a/brain/knowledge/pieces-engine/pieces.md b/brain/knowledge/pieces-engine/pieces.md index 45fbcc3791fe..0486b0ede17b 100644 --- a/brain/knowledge/pieces-engine/pieces.md +++ b/brain/knowledge/pieces-engine/pieces.md @@ -24,6 +24,9 @@ The metadata catalog of automation integrations ("pieces") — each a named inte - Install and sync also enqueue a tool-search reindex, but only when `isToolSearchEnabled()`; no-op otherwise. - `delete` removes all versions sharing the name on that platform, and only for `CUSTOM` pieces the caller owns. - **A piece silently vanishes from the list when its `minimumSupportedRelease` is ahead of the root `package.json` version.** `fetchLatestPieces` filters every piece through `isSupportedRelease(apVersionUtil.getCurrentRelease(), piece)`. Pieces are routinely merged targeting the *next* release, so on `main` a couple dozen are invisible locally until the version bump lands. No warning is logged — it just isn't there. +- **DynamicProperties clears its value before it knows the new schema, so the merge source must be a snapshot.** `DynamicPropertiesImplementation` re-fetches the child schema on every refresher change, clearing the form value synchronously and re-populating it in the mutation callback. The merge source for `getDefaultValueForProperties` has to be a `lastKnownValue` ref captured *before* the clear — reading `form.getValues()` in the callback sees the cleared `null` and defaults every child (GIT-1514). The snapshot must be spread-cloned: RHF `getValues(name)` hands back the live object and the clear's `setValue(...child, null)` mutates it in place. Guard the ref with `isNil` so it survives rapid successive changes, where later effect runs already observe `null`. +- `DynamicPropertiesContext` tracks loading by property name only, so two in-flight requests for the same property let the first completion clear the flag for both — briefly re-enabling Test Step while the value is still cleared. +- **The frontend `POST /v1/pieces/options` client only rejects for DYNAMIC.** `piecesApi.options` (`packages/web/src/features/pieces/api/`) catches DROPDOWN failures, toasts, and *resolves* with a disabled-dropdown fallback — so for dropdowns every error path wired onto that mutation is dead: `usePieceOptions`' `onError` handlers, its `retry: 1`, and the `if (error) throw error` into `DynamicPropertiesErrorBoundary`. DYNAMIC must rethrow: a swallowed failure arrives as a *successful* empty schema, which resets the property's children to defaults and gets persisted by step-settings autosave. - **`AP_DEV_PIECES` shadows the DB registry copy by name**, so a dev piece failing the release gate removes the piece *entirely* rather than falling back to the published version. Dropping the name from `AP_DEV_PIECES` (or bumping the local root `package.json`) brings it back. ### Key files diff --git a/packages/web/src/app/builder/piece-properties/dynamic-piece-property.tsx b/packages/web/src/app/builder/piece-properties/dynamic-piece-property.tsx index 680fd6db7ec0..80b553eb617f 100644 --- a/packages/web/src/app/builder/piece-properties/dynamic-piece-property.tsx +++ b/packages/web/src/app/builder/piece-properties/dynamic-piece-property.tsx @@ -11,6 +11,7 @@ import { useDeepCompareEffectNoCheck } from 'use-deep-compare-effect'; import { useBuilderStateContext } from '@/app/builder/builder-hooks'; import { SkeletonList } from '@/components/ui/skeleton'; +import { internalErrorToast } from '@/components/ui/sonner'; import { piecesHooks, formUtils } from '@/features/pieces'; import { authenticationSession } from '@/lib/authentication-session'; @@ -58,6 +59,10 @@ const DynamicPropertiesImplementation = React.memo( }, {}); const previousRefresherValues = useRef>(refresherValues); + const lastKnownValue = useRef | undefined>( + undefined, + ); + const optionsRequestId = useRef(0); const { propertyLoadingFinished, propertyLoadingStarted } = useContext( DynamicPropertiesContext, ); @@ -73,6 +78,7 @@ const DynamicPropertiesImplementation = React.memo( }, onError: (error) => { console.error(error); + internalErrorToast(); propertyLoadingFinished(props.propertyName); }, onSuccess: () => { @@ -109,10 +115,26 @@ const DynamicPropertiesImplementation = React.memo( ); }; useDeepCompareEffectNoCheck(() => { + const propertyPath = prependPrefixToPropertyName({ + propertyName: props.propertyName, + prefix: propertyPrefix, + }); + const currentValue = form.getValues(propertyPath); + if (!isNil(currentValue)) { + lastKnownValue.current = { ...currentValue }; + } if (!deepEqual(previousRefresherValues.current, refresherValues)) { clearPropertyValue(); } previousRefresherValues.current = refresherValues; + const requestId = ++optionsRequestId.current; + const restoreLastKnownValue = () => { + if (!isNil(lastKnownValue.current)) { + form.setValue(propertyPath, lastKnownValue.current, { + shouldValidate: true, + }); + } + }; mutate( { request: { @@ -129,25 +151,19 @@ const DynamicPropertiesImplementation = React.memo( }, { onSuccess: (response) => { - const currentValue = form.getValues( - prependPrefixToPropertyName({ - propertyName: props.propertyName, - prefix: propertyPrefix, - }), - ); + if (requestId !== optionsRequestId.current) { + return; + } const defaultValue = formUtils.getDefaultValueForProperties({ props: response.options, - existingInput: currentValue ?? {}, + existingInput: lastKnownValue.current ?? {}, propertySettings: props.propertySettings ?? {}, }); setPropertyMap(response.options); const schemaWithoutDropdownOptions = removeOptionsFromDropdownPropertiesSchema(response.options); props.updateFormSchema?.( - prependPrefixToPropertyName({ - propertyName: props.propertyName, - prefix: propertyPrefix, - }), + propertyPath, schemaWithoutDropdownOptions, ); @@ -158,17 +174,16 @@ const DynamicPropertiesImplementation = React.memo( form, ); } - form.setValue( - prependPrefixToPropertyName({ - propertyName: props.propertyName, - prefix: propertyPrefix, - }), - defaultValue, - { - shouldValidate: true, - shouldDirty: true, - }, - ); + form.setValue(propertyPath, defaultValue, { + shouldValidate: true, + shouldDirty: true, + }); + }, + onError: () => { + if (requestId !== optionsRequestId.current) { + return; + } + restoreLastKnownValue(); }, }, ); diff --git a/packages/web/src/features/pieces/api/pieces-api.ts b/packages/web/src/features/pieces/api/pieces-api.ts index 10915232a405..0eba9c1fcb02 100644 --- a/packages/web/src/features/pieces/api/pieces-api.ts +++ b/packages/web/src/features/pieces/api/pieces-api.ts @@ -4,7 +4,6 @@ import { PiecePackageInformation, PropertyType, ExecutePropsResult, - InputPropertyMap, } from '@activepieces/pieces-framework'; import { AddPieceRequestBody, @@ -46,13 +45,11 @@ export const piecesApi = { return api .post>(`/v1/pieces/options`, request) .catch((error) => { + if (propertyType === PropertyType.DYNAMIC) { + throw error; + } console.error(error); internalErrorToast(); - const defaultStateForDynamicProperty: ExecutePropsResult = - { - options: {} as InputPropertyMap, - type: PropertyType.DYNAMIC, - }; const defaultStateForDropdownProperty: ExecutePropsResult = { options: { @@ -64,11 +61,7 @@ export const piecesApi = { }, type: PropertyType.DROPDOWN, }; - return ( - propertyType === PropertyType.DYNAMIC - ? defaultStateForDynamicProperty - : defaultStateForDropdownProperty - ) as ExecutePropsResult; + return defaultStateForDropdownProperty as ExecutePropsResult; }); }, syncFromCloud() { diff --git a/packages/web/test/app/builder/piece-properties/dynamic-piece-property.test.tsx b/packages/web/test/app/builder/piece-properties/dynamic-piece-property.test.tsx new file mode 100644 index 000000000000..61572c3e2fc6 --- /dev/null +++ b/packages/web/test/app/builder/piece-properties/dynamic-piece-property.test.tsx @@ -0,0 +1,262 @@ +/** + * @vitest-environment jsdom + */ +import { + PiecePropertyMap, + Property, +} from '@activepieces/pieces-framework'; +import * as React from 'react'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { + FormProvider, + useForm, + type FieldValues, + type UseFormReturn, +} from 'react-hook-form'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('i18next', () => ({ t: (key: string) => key })); + +vi.mock('@/lib/authentication-session', () => ({ + authenticationSession: { getProjectId: () => 'test-project' }, +})); + +vi.mock('@/app/builder/builder-hooks', () => ({ + useBuilderStateContext: ( + selector: (state: Record) => unknown, + ) => + selector({ + flowVersion: { id: 'flow-version-id', flowId: 'flow-id' }, + readonly: false, + }), +})); + +vi.mock('@/components/ui/skeleton', () => ({ + SkeletonList: () => null, +})); + +vi.mock( + '@/app/builder/piece-properties/dynamic-piece-properties-error-boundary', + () => ({ + DynamicPropertiesErrorBoundary: ({ children }: React.PropsWithChildren) => + children, + }), +); + +vi.mock('@/app/builder/piece-properties/generic-properties-form', () => ({ + GenericPropertiesForm: () => null, +})); + +declare global { + var IS_REACT_ACT_ENVIRONMENT: boolean; +} + +globalThis.IS_REACT_ACT_ENVIRONMENT = true; + +type MutateOptions = { + onSuccess: (response: { options: PiecePropertyMap }) => void; + onError: () => void; +}; + +const mutateCalls: MutateOptions[] = []; + +vi.mock('@/features/pieces', async () => { + const { formUtils } = await vi.importActual< + typeof import('@/features/pieces/utils/form-utils') + >('@/features/pieces/utils/form-utils'); + return { + formUtils, + piecesHooks: { + usePieceOptions: () => ({ + mutate: (_request: unknown, options: MutateOptions) => { + mutateCalls.push(options); + }, + isPending: false, + }), + }, + }; +}); + +import { DynamicProperties } from '@/app/builder/piece-properties/dynamic-piece-property'; + +const shortTextSchema = (fieldNames: string[]): PiecePropertyMap => { + const schema: PiecePropertyMap = {}; + fieldNames.forEach((fieldName) => { + schema[fieldName] = Property.ShortText({ + displayName: fieldName, + required: false, + }); + }); + return schema; +}; + +let formInstance: UseFormReturn | undefined; + +const Harness = () => { + const form = useForm({ + defaultValues: { + settings: { + input: { + items: ['a'], + }, + }, + }, + }); + formInstance = form; + return ( + + + + ); +}; + +describe('DynamicProperties refresher change', () => { + let root: Root | undefined; + let container: HTMLDivElement | undefined; + + afterEach(() => { + act(() => root?.unmount()); + container?.remove(); + mutateCalls.length = 0; + formInstance = undefined; + }); + + const mount = () => { + container = document.createElement('div'); + document.body.appendChild(container); + act(() => { + root = createRoot(container!); + root.render(); + }); + }; + + const resolveOptions = (options: PiecePropertyMap) => { + const call = mutateCalls[mutateCalls.length - 1]; + act(() => call.onSuccess({ options })); + }; + + it('preserves user-entered values when a refresher gains an array item', () => { + mount(); + expect(mutateCalls).toHaveLength(1); + resolveOptions(shortTextSchema(['firstName', 'lastName'])); + + act(() => { + formInstance!.setValue('settings.input.fields.firstName', 'John'); + formInstance!.setValue('settings.input.fields.lastName', 'Doe'); + }); + + act(() => formInstance!.setValue('settings.input.items', ['a', 'b'])); + expect(mutateCalls).toHaveLength(2); + expect(formInstance!.getValues('settings.input.fields')).toBeNull(); + + resolveOptions(shortTextSchema(['firstName', 'lastName'])); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: 'John', + lastName: 'Doe', + }); + }); + + it('drops values for keys absent from the new schema', () => { + mount(); + resolveOptions(shortTextSchema(['firstName', 'lastName'])); + + act(() => { + formInstance!.setValue('settings.input.fields.firstName', 'John'); + formInstance!.setValue('settings.input.fields.lastName', 'Doe'); + }); + + act(() => formInstance!.setValue('settings.input.items', ['a', 'b'])); + resolveOptions(shortTextSchema(['firstName', 'email'])); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: 'John', + email: '', + }); + }); + + it('preserves values across rapid successive refresher changes', () => { + mount(); + resolveOptions(shortTextSchema(['firstName'])); + + act(() => + formInstance!.setValue('settings.input.fields.firstName', 'John'), + ); + + act(() => formInstance!.setValue('settings.input.items', ['a', 'b'])); + act(() => formInstance!.setValue('settings.input.items', ['a', 'b', 'c'])); + expect(mutateCalls).toHaveLength(3); + + resolveOptions(shortTextSchema(['firstName'])); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: 'John', + }); + }); + + it('applies defaults on first load when no prior value exists', () => { + mount(); + resolveOptions(shortTextSchema(['firstName'])); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: '', + }); + }); + + it('ignores a stale response resolving after a newer one', () => { + mount(); + resolveOptions(shortTextSchema(['firstName'])); + + act(() => + formInstance!.setValue('settings.input.fields.firstName', 'John'), + ); + + act(() => formInstance!.setValue('settings.input.items', ['a', 'b'])); + act(() => formInstance!.setValue('settings.input.items', ['a', 'b', 'c'])); + expect(mutateCalls).toHaveLength(3); + + act(() => + mutateCalls[2].onSuccess({ + options: shortTextSchema(['firstName', 'email']), + }), + ); + act(() => + mutateCalls[1].onSuccess({ options: shortTextSchema(['firstName']) }), + ); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: 'John', + email: '', + }); + }); + + it('restores values when the options request fails', () => { + mount(); + resolveOptions(shortTextSchema(['firstName'])); + + act(() => + formInstance!.setValue('settings.input.fields.firstName', 'John'), + ); + + act(() => formInstance!.setValue('settings.input.items', ['a', 'b'])); + expect(formInstance!.getValues('settings.input.fields')).toBeNull(); + + act(() => mutateCalls[1].onError()); + + expect(formInstance!.getValues('settings.input.fields')).toEqual({ + firstName: 'John', + }); + }); +}); diff --git a/packages/web/test/features/pieces/pieces-api.test.ts b/packages/web/test/features/pieces/pieces-api.test.ts new file mode 100644 index 000000000000..51c70414a778 --- /dev/null +++ b/packages/web/test/features/pieces/pieces-api.test.ts @@ -0,0 +1,43 @@ +import { PropertyType } from '@activepieces/pieces-framework'; +import { PieceOptionRequest } from '@activepieces/shared'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('i18next', () => ({ t: (key: string) => key })); + +const internalErrorToast = vi.fn(); +vi.mock('@/components/ui/sonner', () => ({ + internalErrorToast: () => internalErrorToast(), +})); + +const post = vi.fn(); +vi.mock('@/lib/api', () => ({ api: { post: () => post() } })); + +import { piecesApi } from '@/features/pieces/api/pieces-api'; + +const request = {} as PieceOptionRequest; + +describe('piecesApi.options', () => { + beforeEach(() => { + post.mockReset(); + internalErrorToast.mockReset(); + }); + + it('rejects for DYNAMIC so callers can restore the cleared value', async () => { + const failure = new Error('boom'); + post.mockRejectedValue(failure); + + await expect( + piecesApi.options(request, PropertyType.DYNAMIC), + ).rejects.toBe(failure); + }); + + it('resolves a disabled dropdown state for DROPDOWN', async () => { + post.mockRejectedValue(new Error('boom')); + + const result = await piecesApi.options(request, PropertyType.DROPDOWN); + + expect(result.type).toBe(PropertyType.DROPDOWN); + expect(result.options.disabled).toBe(true); + expect(internalErrorToast).toHaveBeenCalledTimes(1); + }); +}); From 9bbe5e83659b31a3c53f00647390f3245741eeee Mon Sep 17 00:00:00 2001 From: Chaker Atallah <74781393+MrChaker@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:29:37 +0100 Subject: [PATCH 6/6] fix: bun lock file (#14908) Co-authored-by: Amr Elmohamady Co-authored-by: Claude Opus 4.6 (1M context) --- bun.lock | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bun.lock b/bun.lock index ff3ecf1f1d7c..2f26e9ac1bcf 100644 --- a/bun.lock +++ b/bun.lock @@ -162,7 +162,7 @@ }, "packages/core/shared": { "name": "@activepieces/shared", - "version": "0.138.0", + "version": "0.138.1", "dependencies": { "@activepieces/core-execution": "workspace:*", "@activepieces/core-formula": "workspace:*", @@ -7678,6 +7678,20 @@ "tslib": "2.6.2", }, }, + "packages/pieces/community/ringcentral": { + "name": "@activepieces/piece-ringcentral", + "version": "0.0.1", + "dependencies": { + "@activepieces/core-piece-types": "workspace:*", + "@activepieces/core-utils": "workspace:*", + "@activepieces/pieces-common": "workspace:*", + "@activepieces/pieces-framework": "workspace:*", + }, + "devDependencies": { + "tslib": "2.6.2", + "vitest": "3.2.6", + }, + }, "packages/pieces/community/robolly": { "name": "@activepieces/piece-robolly", "version": "0.1.7", @@ -12176,6 +12190,8 @@ "@activepieces/piece-returning-ai": ["@activepieces/piece-returning-ai@workspace:packages/pieces/community/returning-ai"], + "@activepieces/piece-ringcentral": ["@activepieces/piece-ringcentral@workspace:packages/pieces/community/ringcentral"], + "@activepieces/piece-robolly": ["@activepieces/piece-robolly@workspace:packages/pieces/community/robolly"], "@activepieces/piece-roe-ai": ["@activepieces/piece-roe-ai@workspace:packages/pieces/community/roe-ai"],