FOUR-32613 - Realtime (or at least faster) script executor

ProcessMaker/Enums/ScriptExecutorType.php

@@ -7,4 +7,10 @@ enum ScriptExecutorType:string
     case System = 'system';
     case Custom = 'custom';
     case Duplicate = 'duplicate';
+    case Realtime = 'realtime';
+
+    public function isCustomOrRealtime(): bool
+    {
+        return $this === self::Custom || $this === self::Realtime;
+    }
 }

ProcessMaker/Events/ScriptResponseEvent.php

@@ -25,18 +25,21 @@ class ScriptResponseEvent implements ShouldBroadcastNow
 
     public $nonce;
 
+    public $duration;
+
     /**
      * Create a new event instance.
      *
      * @return void
      */
-    public function __construct(User $user, $status, array $response, $watcher = null, $nonce = null)
+    public function __construct(User $user, $status, array $response, $watcher = null, $nonce = null, $duration = null)
     {
         $this->userId = $user->id;
         $this->status = $status;
         $this->response = $response;
         $this->watcher = $watcher;
         $this->nonce = $nonce;
+        $this->duration = $duration;
     }
 
     /**
@@ -75,6 +78,7 @@ public function broadcastWith()
             'watcher' => $this->watcher,
             'response' => $response,
             'nonce' => $this->nonce,
+            'duration' => $this->duration,
         ];
     }
 }

ProcessMaker/Http/Controllers/Admin/ScriptExecutorController.php

@@ -8,15 +8,20 @@
 
 class ScriptExecutorController extends Controller
 {
-    public function index(Request $request)
+    public function index(Request $request, ScriptMicroserviceService $service)
     {
         if (!config('app.custom_executors')) {
             abort(404);
         }
 
+        $scriptMicroserviceEnabled = config('script-runner-microservice.enabled');
+
         return view('admin.script-executors.index',
             [
-                'script_microservice_enabled' => config('script-runner-microservice.enabled'),
+                'script_microservice_enabled' => $scriptMicroserviceEnabled,
+                'script_microservice_tenant_id' => $scriptMicroserviceEnabled
+                    ? $service->getInstanceUuid()
+                    : null,
             ]);
     }
 }

ProcessMaker/Http/Controllers/Api/ScriptController.php

@@ -4,6 +4,7 @@
 
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Cache;
+use ProcessMaker\Enums\ScriptExecutorType;
 use ProcessMaker\Events\ScriptCreated;
 use ProcessMaker\Events\ScriptDeleted;
 use ProcessMaker\Events\ScriptDuplicated;
@@ -178,7 +179,7 @@ public function index(Request $request)
      *
      *     @OA\Response(
      *         response=200,
-     *         description="success if the script was queued",
+     *         description="The queued status or realtime execution response",
      *         ),
      *     ),
      * )
@@ -190,7 +191,12 @@ public function preview(Request $request, Script $script)
         $code = $request->get('code');
         $nonce = $request->get('nonce');
 
-        TestScript::dispatch($script, $request->user(), $code, $data, $config, $nonce)->onQueue('bpmn');
+        $job = TestScript::dispatch($script, $request->user(), $code, $data, $config, $nonce);
+        if ($script->scriptExecutor->type === ScriptExecutorType::Realtime) {
+            $job->onConnection('redis-realtime')->onQueue('realtime');
+        } else {
+            $job->onQueue('bpmn');
+        }
 
         return ['status' => 'success'];
     }
@@ -249,7 +255,12 @@ public function execute(Request $request, ...$scriptKey)
         if ($request->get('sync') === true) {
             return (new ExecuteScript($script, $request->user(), $code, $data, $watcher, $config, true))->handle();
         } else {
-            ExecuteScript::dispatch($script, $request->user(), $code, $data, $watcher, $config)->onQueue('bpmn');
+            $job = ExecuteScript::dispatch($script, $request->user(), $code, $data, $watcher, $config);
+            if ($script->scriptExecutor->type === ScriptExecutorType::Realtime) {
+                $job->onConnection('redis-realtime')->onQueue('realtime');
+            } else {
+                $job->onQueue('bpmn');
+            }
         }
 
         return ['status' => 'success', 'key' => $watcher];

ProcessMaker/Http/Controllers/Api/ScriptExecutorController.php

@@ -4,6 +4,7 @@
 
 use Illuminate\Auth\Access\AuthorizationException;
 use Illuminate\Database\Eloquent\ModelNotFoundException;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Http\Request;
 use Illuminate\Validation\ValidationException;
 use ProcessMaker\Enums\ScriptExecutorType;
@@ -124,7 +125,15 @@ public function store(Request $request, ScriptMicroserviceService $service)
             ScriptExecutorCreated::dispatch($scriptExecutor->getAttributes());
             BuildScriptExecutor::dispatch($scriptExecutor->id, $request->user()->id);
         } else {
-            $service->createCustomExecutor($scriptExecutor);
+            try {
+                $service->createCustomExecutor($scriptExecutor);
+            } catch (RequestException $e) {
+                // The remote executor was rejected, so keeping the local record would leave
+                // an executor that can never be built.
+                $scriptExecutor->delete();
+
+                $this->throwMicroserviceError($e);
+            }
         }
 
         return ['status' => 'started', 'uuid' => $scriptExecutor->uuid, 'id' => $scriptExecutor->id];
@@ -188,8 +197,12 @@ public function update(Request $request, ScriptExecutor $scriptExecutor, ScriptM
             $request->only($scriptExecutor->getFillable())
         );
 
-        if (config('script-runner-microservice.enabled') && $scriptExecutor->type == ScriptExecutorType::Custom) {
-            $service->updateCustomExecutor($scriptExecutor);
+        if (config('script-runner-microservice.enabled') && $scriptExecutor->type?->isCustomOrRealtime()) {
+            try {
+                $service->updateCustomExecutor($scriptExecutor);
+            } catch (RequestException $e) {
+                $this->throwMicroserviceError($e);
+            }
         } else {
             if (!empty($scriptExecutor->getChanges())) {
                 ScriptExecutorUpdated::dispatch($scriptExecutor->id, $original, $scriptExecutor->getChanges());
@@ -274,6 +287,28 @@ public function delete(Request $request, ScriptExecutor $scriptExecutor, ScriptM
         return ['status' => 'done'];
     }
 
+    /**
+     * Report a script microservice rejection on the form, since the build itself
+     * is only reported over the websocket channel.
+     *
+     * @param RequestException $e Failed script microservice response
+     *
+     * @return never
+     *
+     * @throws ValidationException|RequestException
+     */
+    private function throwMicroserviceError(RequestException $e): never
+    {
+        if (!$e->response->clientError()) {
+            throw $e;
+        }
+
+        $detail = $e->response->json('detail');
+        $message = is_string($detail) && $detail !== '' ? $detail : $e->getMessage();
+
+        throw ValidationException::withMessages(['language' => [$message]]);
+    }
+
     private function checkAuth($request)
     {
         if (!config('app.custom_executors')) {
@@ -381,11 +416,30 @@ public function availableLanguages()
                 $languages[] = [
                     'value' => $key,
                     'text' => $config['name'],
+                    'language' => $key,
+                    'realtime' => false,
                     'initDockerfile' => ScriptExecutor::initDockerfile($key),
+                    'configExample' => '',
                 ];
             }
         }
 
+        foreach (['php', 'python', 'javascript'] as $language) {
+            $dockerfilePath = resource_path("script-executors/realtime/{$language}.Dockerfile");
+            if (!file_exists($dockerfilePath)) {
+                continue;
+            }
+            $label = $language === 'javascript' ? 'nodejs (realtime)' : "{$language} (realtime)";
+            $languages[] = [
+                'value' => "{$language}-realtime",
+                'text' => $label,
+                'language' => $language,
+                'realtime' => true,
+                'initDockerfile' => '',
+                'configExample' => file_get_contents($dockerfilePath),
+            ];
+        }
+
         return ['languages' => $languages];
     }
 }

ProcessMaker/Jobs/ErrorHandling.php

@@ -83,7 +83,8 @@ private function requeue($job)
             );
         }
         $newJob->delay($this->retryWaitTime());
-        $newJob->onQueue('bpmn');
+        $newJob->onConnection($job->connection);
+        $newJob->onQueue($job->queue ?? 'bpmn');
         dispatch($newJob);
     }
 

ProcessMaker/Jobs/TestScript.php

@@ -7,10 +7,12 @@
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
 use ProcessMaker\Enums\ScriptExecutorType;
 use ProcessMaker\Events\ScriptResponseEvent;
 use ProcessMaker\Models\Script;
 use ProcessMaker\Models\User;
+use ProcessMaker\Services\ScriptMicroserviceService;
 use Throwable;
 
 class TestScript implements ShouldQueue
@@ -35,8 +37,8 @@ class TestScript implements ShouldQueue
     /**
      * Create a new job instance to execute a script.
      *
-     * @param ProcessMaker\Models\Script $script
-     * @param ProcessMaker\Models\User $current_user
+     * @param Script $script
+     * @param User $current_user
      * @param string $code
      * @param array $data
      * @param array $configuration
@@ -58,24 +60,31 @@ public function __construct(Script $script, User $current_user, $code, array $da
      */
     public function handle()
     {
+        $startTime = microtime(true);
         try {
             // Just set the code but do not save the object (preview only)
             $this->script->code = $this->code;
             $metadata = [
                 'nonce' => $this->nonce,
                 'current_user' => $this->current_user?->id,
+                'start_time' => microtime(true),
             ];
             $response = $this->script->runScript($this->data, $this->configuration, '', null, 0, $metadata);
-            \Log::debug('Response from runScript: ' . print_r($response, true));
+            Log::debug('Response from runScript: ' . print_r($response, true));
 
-            if (!config('script-runner-microservice.enabled')) {
-                $this->sendResponse(200, $response);
+            if ($this->script->scriptExecutor->type === ScriptExecutorType::Realtime) {
+                // Realtime executors return the microservice payload synchronously rather than
+                // posting back to the callback endpoint, so format it the same way here.
+                $formatted = (new ScriptMicroserviceService())->formatPreviewResponse($response);
+                $this->sendResponse($formatted['status'], $formatted['output'], (microtime(true) - $startTime));
+            } elseif (!config('script-runner-microservice.enabled')) {
+                $this->sendResponse(200, $response, (microtime(true) - $startTime));
             }
         } catch (Throwable $exception) {
             $this->sendResponse(500, [
                 'exception' => get_class($exception),
                 'message' => $exception->getMessage(),
-            ]);
+            ], (microtime(true) - $startTime));
         }
     }
 
@@ -85,8 +94,8 @@ public function handle()
      * @param int $status
      * @param array $response
      */
-    private function sendResponse($status, array $response)
+    private function sendResponse($status, array $response, float $duration)
     {
-        event(new ScriptResponseEvent($this->current_user, $status, $response, null, $this->nonce));
+        event(new ScriptResponseEvent($this->current_user, $status, $response, null, $this->nonce, $duration));
     }
 }

ProcessMaker/Models/Script.php

@@ -179,6 +179,7 @@ public function runScript(array $data, array $config, $tokenId = '', $timeout =
         if (!$user) {
             throw new ConfigurationException('A user is required to run scripts');
         }
+        $metadata['start_time'] = microtime(true);
 
         return $runner->run($this->code, $data, $config, $timeout, $user, $sync, $metadata);
     }

ProcessMaker/Models/ScriptExecutor.php

@@ -32,6 +32,7 @@
  *   @OA\Property(property="language", type="string"),
  *   @OA\Property(property="config", type="string"),
  *   @OA\Property(property="is_system", type="boolean"),
+ *   @OA\Property(property="type", type="string"),
  * ),
  * @OA\Schema(
  *   schema="scriptExecutors",
@@ -177,6 +178,12 @@ public static function rules($existing = null)
             });
         }
 
+        // Realtime executors may use php/python/javascript even if a package is not installed
+        $allowedLanguages = array_values(array_unique(array_merge(
+            $allowedLanguages,
+            ['php', 'python', 'javascript']
+        )));
+
         return [
             'title' => 'required',
             'language' => [

ProcessMaker/Nayra/Managers/WorkflowManagerDefault.php

@@ -8,6 +8,7 @@
 use ProcessMaker\BpmnEngine;
 use ProcessMaker\Contracts\ServiceTaskImplementationInterface;
 use ProcessMaker\Contracts\WorkflowManagerInterface;
+use ProcessMaker\Enums\ScriptExecutorType;
 use ProcessMaker\Jobs\BoundaryEvent;
 use ProcessMaker\Jobs\CallProcess;
 use ProcessMaker\Jobs\CatchEvent;
@@ -23,6 +24,7 @@
 use ProcessMaker\Models\Process as Definitions;
 use ProcessMaker\Models\ProcessRequest;
 use ProcessMaker\Models\ProcessRequestToken as Token;
+use ProcessMaker\Models\Script;
 use ProcessMaker\Nayra\Contracts\Bpmn\BoundaryEventInterface;
 use ProcessMaker\Nayra\Contracts\Bpmn\EntityInterface;
 use ProcessMaker\Nayra\Contracts\Bpmn\EventDefinitionInterface;
@@ -231,7 +233,14 @@ public function runScripTask(ScriptTaskInterface $scriptTask, Token $token)
 
         $instance = $token->processRequest;
         $process = $instance->process;
-        RunScriptTask::dispatch($process, $instance, $token, [])->onQueue('bpmn');
+        $job = new RunScriptTask($process, $instance, $token, []);
+        $script = Script::find($scriptTask->getProperty('scriptRef'));
+
+        if ($script?->scriptExecutor?->type === ScriptExecutorType::Realtime) {
+            $job->onConnection('redis-realtime')->onQueue('realtime');
+        }
+
+        dispatch($job);
     }
 
     /**

ProcessMaker/ScriptRunners/ScriptMicroserviceRunner.php

@@ -40,7 +40,7 @@ public function run($code, array $data, array $config, $timeout, $user, $sync, $
         $scriptRunner = $this->service->getScriptRunner(
             $this->language,
             $this->script->scriptExecutor->uuid,
-            $this->script->scriptExecutor->type === ScriptExecutorType::Custom
+            $this->script->scriptExecutor->type?->isCustomOrRealtime()
         );
 
         if (!$scriptRunner) {
@@ -62,7 +62,8 @@ public function run($code, array $data, array $config, $timeout, $user, $sync, $
             'callback_token' => $environmentVariables['API_TOKEN'],
             'debug' => true,
             'timeout' => $timeout,
-            'sync' => $sync,
+            // Realtime executors always run synchronously (service ignores sync too)
+            'sync' => $this->script->scriptExecutor->type === ScriptExecutorType::Realtime ? true : $sync,
         ];
 
         Log::debug('Payload: ' . print_r($payload, true));