[pull] main from activepieces:main

packages/pieces/community/jira-cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activepieces/piece-jira-cloud",
-  "version": "0.3.10",
+  "version": "0.3.11",
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "dependencies": {

packages/pieces/community/jira-cloud/src/lib/common/index.ts

@@ -70,6 +70,29 @@ export async function getPriorities({ auth }: { auth: JiraAuth }) {
 	return response.body as any[];
 }
 
+export async function sanitizeJqlQuery({ auth, jql }: { auth: JiraAuth; jql: string }): Promise<string> {
+	const sanitizeResult = (
+		await sendJiraRequest({
+			auth: auth,
+			url: 'jql/sanitize',
+			method: HttpMethod.POST,
+			body: {
+				queries: [
+					{
+						query: jql,
+					},
+				],
+			},
+		})
+	).body as {
+		queries: {
+			initialQuery: string;
+			sanitizedQuery: string;
+		}[];
+	};
+	return sanitizeResult.queries[0].sanitizedQuery;
+}
+
 export async function executeJql({
 	auth,
 	jql,
@@ -87,29 +110,7 @@ export async function executeJql({
 	queryParams?: QueryParams;
 	body?: HttpMessageBody;
 }) {
-	let reqJql = jql;
-	if (sanitizeJql) {
-		const sanitizeResult = (
-			await sendJiraRequest({
-				auth: auth,
-				url: 'jql/sanitize',
-				method: HttpMethod.POST,
-				body: {
-					queries: [
-						{
-							query: jql,
-						},
-					],
-				},
-			})
-		).body as {
-			queries: {
-				initialQuery: string;
-				sanitizedQuery: string;
-			}[];
-		};
-		reqJql = sanitizeResult.queries[0].sanitizedQuery;
-	}
+	const reqJql = sanitizeJql ? await sanitizeJqlQuery({ auth, jql }) : jql;
 
 	const response = await sendJiraRequest({
 		auth,

packages/pieces/community/jira-cloud/src/lib/common/polling.ts

@@ -0,0 +1,233 @@
+// https://developer.atlassian.com/cloud/jira/platform/search-and-reconcile/
+import { DEDUPE_KEY_PROPERTY, Store, isNil } from '@activepieces/pieces-framework';
+import { JiraAuth } from '../../auth';
+import { sanitizeJqlQuery, searchIssuesByJql } from './index';
+import { JiraSearchResponse } from './types';
+
+const LOOKBACK_MS = 15 * 60 * 1000;
+const MAX_RESULTS = 1000;
+const MAX_PAGES = 10;
+const MAX_LEDGER_ENTRIES = 2000;
+const TEST_ITEMS_LIMIT = 5;
+const STATE_STORE_KEY = 'pollingState';
+
+function toRelativeJqlDate({ sinceEpochMS, now }: { sinceEpochMS: number; now: number }): string {
+	return `-${Math.max(Math.ceil((now - sinceEpochMS) / 60_000) + 1, 1)}m`;
+}
+
+function stripTrailingOrderBy(jql: string): string {
+	let quote: '"' | "'" | null = null;
+	let orderByIndex = -1;
+
+	for (let index = 0; index < jql.length && orderByIndex === -1; index++) {
+		const char = jql[index];
+
+		if (quote) {
+			if (char === '\\') {
+				index += 1;
+			} else if (char === quote) {
+				quote = null;
+			}
+			continue;
+		}
+
+		if (char === '"' || char === "'") {
+			quote = char;
+			continue;
+		}
+
+		if (/\s/.test(char) && /^\s+order\s+by\b/i.test(jql.slice(index))) {
+			orderByIndex = index;
+		}
+	}
+
+	return (orderByIndex === -1 ? jql : jql.slice(0, orderByIndex)).trim();
+}
+
+async function composeJql({
+	auth,
+	propsValue,
+	since,
+	direction,
+}: {
+	auth: JiraAuth;
+	propsValue: JiraPollingContext['propsValue'];
+	since: string | null;
+	direction: 'ASC' | 'DESC';
+}): Promise<string> {
+	const userJql = isNil(propsValue.jql) ? '' : stripTrailingOrderBy(propsValue.jql);
+	const scope =
+		userJql.length === 0
+			? null
+			: `(${propsValue.sanitizeJql ? await sanitizeJqlQuery({ auth, jql: userJql }) : userJql})`;
+	const conditions = [scope, isNil(since) ? null : `updated > '${since}'`].filter(
+		(condition): condition is string => !isNil(condition),
+	);
+	const orderBy = `ORDER BY updated ${direction}`;
+	return conditions.length === 0 ? orderBy : `${conditions.join(' AND ')} ${orderBy}`;
+}
+
+async function fetchIssues({
+	auth,
+	jql,
+	maxResults,
+	maxPages = 1,
+}: {
+	auth: JiraAuth;
+	jql: string;
+	maxResults: number;
+	maxPages?: number;
+}): Promise<{ items: PollingItem[]; truncated: boolean }> {
+	const issues: JiraSearchResponse['issues'] = [];
+	let nextPageToken: string | undefined;
+	let truncated = false;
+
+	for (let page = 0; page < maxPages; page++) {
+		const response: JiraSearchResponse = await searchIssuesByJql({
+			auth,
+			jql,
+			maxResults,
+			sanitizeJql: false,
+			nextPageToken,
+		});
+		issues.push(...(response.issues ?? []));
+		nextPageToken = response.nextPageToken;
+
+		if (isNil(nextPageToken)) {
+			break;
+		}
+		if (page === maxPages - 1) {
+			truncated = true;
+		}
+	}
+
+	return { items: toItems(issues), truncated };
+}
+
+function toItems(issues: JiraSearchResponse['issues']): PollingItem[] {
+	return issues
+		.flatMap((issue) => {
+			const epochMilliSeconds = Date.parse(issue?.fields?.updated);
+			return Number.isFinite(epochMilliSeconds)
+				? [{ key: `${issue?.id}:${epochMilliSeconds}`, epochMilliSeconds, data: issue }]
+				: [];
+		})
+		.sort((first, second) => first.epochMilliSeconds - second.epochMilliSeconds);
+}
+
+function pruneState({
+	entries,
+	floor,
+	windowStart,
+}: {
+	entries: LedgerEntry[];
+	floor: number;
+	windowStart: number;
+}): PollingState {
+	const insideWindow = entries
+		.filter(([, epoch]) => epoch > windowStart)
+		.sort((first, second) => first[1] - second[1]);
+	const overflow = Math.max(insideWindow.length - MAX_LEDGER_ENTRIES, 0);
+	const dropped = [
+		...entries.filter(([, epoch]) => epoch <= windowStart),
+		...insideWindow.slice(0, overflow),
+	];
+
+	return {
+		windowStart,
+		floor: dropped.reduce(
+			(highest, [, epoch]) => Math.max(highest, epoch),
+			Math.max(floor, windowStart),
+		),
+		entries: insideWindow.slice(overflow),
+	};
+}
+
+function initialState(now: number): PollingState {
+	return { windowStart: now, floor: now, entries: [] };
+}
+
+export const jiraPolling = {
+	async onEnable({ context }: { context: JiraPollingContext }): Promise<void> {
+		const { store, isRepublish } = context;
+		if (isRepublish && !isNil(await store.get<PollingState>(STATE_STORE_KEY))) {
+			return;
+		}
+		await store.put(STATE_STORE_KEY, initialState(Date.now()));
+	},
+
+	async poll({ context }: { context: JiraPollingContext }): Promise<unknown[]> {
+		const { auth, propsValue, store } = context;
+		const now = Date.now();
+		const state = (await store.get<PollingState>(STATE_STORE_KEY)) ?? initialState(now);
+		const windowStart = Math.max(state.windowStart, state.floor);
+
+		const { items, truncated } = await fetchIssues({
+			auth,
+			jql: await composeJql({
+				auth,
+				propsValue,
+				since: toRelativeJqlDate({ sinceEpochMS: windowStart, now }),
+				direction: 'ASC',
+			}),
+			maxResults: MAX_RESULTS,
+			maxPages: MAX_PAGES,
+		});
+
+		const alreadyEmitted = new Set(state.entries.map(([key]) => key));
+		const freshItems = items.filter(
+			(item) => item.epochMilliSeconds > windowStart && !alreadyEmitted.has(item.key),
+		);
+
+		const highestEpoch = items.at(-1)?.epochMilliSeconds;
+		const nextWindowStart = truncated
+			? Math.max(windowStart, isNil(highestEpoch) ? windowStart : highestEpoch - 1)
+			: Math.max(windowStart, now - LOOKBACK_MS);
+
+		await store.put(
+			STATE_STORE_KEY,
+			pruneState({
+				entries: [
+					...state.entries,
+					...freshItems.map((item): LedgerEntry => [item.key, item.epochMilliSeconds]),
+				],
+				floor: state.floor,
+				windowStart: nextWindowStart,
+			}),
+		);
+
+		return freshItems.map((item) => ({ ...item.data, [DEDUPE_KEY_PROPERTY]: item.key }));
+	},
+
+	async test({ context }: { context: JiraPollingContext }): Promise<unknown[]> {
+		const { auth, propsValue } = context;
+		const { items } = await fetchIssues({
+			auth,
+			jql: await composeJql({ auth, propsValue, since: null, direction: 'DESC' }),
+			maxResults: TEST_ITEMS_LIMIT,
+		});
+
+		return items.reverse().map((item) => item.data);
+	},
+};
+
+type JiraPollingContext = {
+	auth: JiraAuth;
+	propsValue: { jql?: string; sanitizeJql?: boolean };
+	store: Store;
+	isRepublish?: boolean;
+};
+
+type PollingItem = {
+	key: string;
+	epochMilliSeconds: number;
+	data: Record<string, unknown>;
+};
+
+type LedgerEntry = [key: string, epochMilliSeconds: number];
+
+type PollingState = {
+	windowStart: number;
+	floor: number;
+	entries: LedgerEntry[];
+};

packages/pieces/community/jira-cloud/src/lib/triggers/updated-issue.ts

@@ -3,36 +3,8 @@ import {
   TriggerStrategy,
   createTrigger,
 } from '@activepieces/pieces-framework';
-import {
-  Polling,
-  DedupeStrategy,
-  pollingHelper,
-} from '@activepieces/pieces-common';
-import { JiraAuth, jiraCloudAuth } from '../../auth';
-import { searchIssuesByJql } from '../common';
-import dayjs from 'dayjs';
-
-const polling: Polling<   JiraAuth,
-  { jql?: string; sanitizeJql?: boolean }
-> = {
-  strategy: DedupeStrategy.TIMEBASED,
-  items: async ({ auth, lastFetchEpochMS, propsValue }) => {
-    const { jql, sanitizeJql } = propsValue;
-    const searchQuery = `${jql ? jql + ' AND ' : ''}updated > '${dayjs(
-      lastFetchEpochMS
-    ).format('YYYY-MM-DD HH:mm')}'`;
-    const response = await searchIssuesByJql({
-      auth,
-      jql: searchQuery,
-      maxResults: 50,
-      sanitizeJql: sanitizeJql ?? false,
-    });
-    return response.issues.map((issue: any) => ({
-      epochMilliSeconds: Date.parse(issue.fields.updated),
-      data: issue,
-    }));
-  },
-};
+import { jiraCloudAuth } from '../../auth';
+import { jiraPolling } from '../common/polling';
 
 export const updatedIssue = createTrigger({
   name: 'updated_issue',
@@ -58,15 +30,15 @@ export const updatedIssue = createTrigger({
   },
   sampleData: {},
   async onEnable(context) {
-    await pollingHelper.onEnable(polling, context);
+    await jiraPolling.onEnable({ context });
   },
-  async onDisable(context) {
-    await pollingHelper.onDisable(polling, context);
+  async onDisable() {
+    return;
   },
   async run(context) {
-    return await pollingHelper.poll(polling, context);
+    return await jiraPolling.poll({ context });
   },
   async test(context) {
-    return await pollingHelper.test(polling, context);
+    return await jiraPolling.test({ context });
   },
 });

packages/web/src/features/agents/agent-timeline/index.tsx

@@ -27,7 +27,7 @@ export const AgentTimeline = ({
   agentResult,
   className = '',
 }: AgentTimelineProps) => {
-  if (isNil(agentResult)) {
+  if (isNil(agentResult) || isNil(agentResult.steps)) {
     return <p>{t('No agent output available')}</p>;
   }
 
@@ -37,7 +37,7 @@ export const AgentTimeline = ({
         <div className="absolute left-2 top-4 bottom-8 w-px bg-border" />
 
         <div className="space-y-7 pb-4">
-          {agentResult.prompt.length > 0 && (
+          {(agentResult.prompt?.length ?? 0) > 0 && (
             <PromptBlock prompt={agentResult.prompt} />
           )}