feat(external-call): prepared-transaction codec for recorded results

.circleci/config.yml

@@ -4,7 +4,7 @@ _canton_docker_executor:
     auth:
         password: $DOCKERHUB_PASSWORD
         username: $DOCKERHUB_USERNAME
-    image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+    image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
 _postgres_docker_executor:
     command: |-
         postgres
@@ -763,7 +763,7 @@ commands:
                       --env CANTON_PROTOCOL_VERSION \
                       --security-opt seccomp=unconfined \
                       << parameters.additional_docker_args >> \
-                      digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c \
+                      digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17 \
                       nix-shell \
                       -I nixpkgs=./nix/nixpkgs.nix \
                       --run 'python3 -c "print (\"r\n\"*1000)" | sbt << parameters.sbt_options >> << parameters.command >> '
@@ -1296,7 +1296,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 16000M
             EXECUTOR_JVM_METASPACE_SIZE: 3200M
@@ -1308,7 +1308,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -1338,7 +1338,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 16000M
             EXECUTOR_JVM_METASPACE_SIZE: 12000M
@@ -1350,7 +1350,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -1380,7 +1380,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 4500M
             EXECUTOR_JVM_METASPACE_SIZE: 1000M
@@ -1392,7 +1392,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 4500M
             EXECUTOR_JVM_METASPACE_SIZE: 1000M
@@ -1403,7 +1403,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -1433,7 +1433,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 2500M
             EXECUTOR_JVM_METASPACE_SIZE: 400M
@@ -1445,7 +1445,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 1200M
             EXECUTOR_JVM_METASPACE_SIZE: 200M
@@ -1457,7 +1457,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
         environment:
             EXECUTOR_JVM_HEAP_SIZE: 6500M
             EXECUTOR_JVM_METASPACE_SIZE: 3100M
@@ -1469,7 +1469,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -1499,7 +1499,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -1530,7 +1530,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: postgres -c max_connections=1000
               environment:
                 NON_STANDARD_POSTGRES: "yes"
@@ -1555,7 +1555,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: postgres -c max_connections=1000
               environment:
                 NON_STANDARD_POSTGRES: "yes"
@@ -1580,7 +1580,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: postgres -c max_connections=1000
               environment:
                 NON_STANDARD_POSTGRES: "yes"
@@ -1605,7 +1605,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: postgres -c max_connections=1000
               environment:
                 NON_STANDARD_POSTGRES: "yes"
@@ -1630,7 +1630,7 @@ executors:
             - auth:
                 password: $DOCKERHUB_PASSWORD
                 username: $DOCKERHUB_USERNAME
-              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+              image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
             - command: |-
                 postgres
                   -c max_connections=1000
@@ -3357,19 +3357,6 @@ workflows:
                         only: /^v.*/
                 requires:
                     - compile
-            - test_data_continuity_dumps:
-                context:
-                    - datadog
-                    - artifactory
-                    - azure-self-cache
-                    - maven-mirror
-                    - da-images
-                filters:
-                    branches:
-                        ignore:
-                            - /^gh-readonly-queue\/main\/.*/
-                requires:
-                    - compile
         max_auto_reruns: 1
         when:
             and:

.circleci/config/@config.yml

@@ -68,7 +68,7 @@ _postgres_non_standard_environment: &postgres_non_standard_environment
   <<: *postgres_environment
   NON_STANDARD_POSTGRES: yes
 _canton_docker_executor: &canton_docker_executor
-  image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c
+  image: digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17
   auth:
     username: $DOCKERHUB_USERNAME
     password: $DOCKERHUB_PASSWORD

.circleci/config/commands/@machine.yml

@@ -50,7 +50,7 @@ run_sbt_command_on_machine:
             --env CANTON_PROTOCOL_VERSION \
             --security-opt seccomp=unconfined \
             << parameters.additional_docker_args >> \
-            digitalasset/canton-build:5950b99b5f66-adbfe542cf31-89dc7fac0e6c \
+            digitalasset/canton-build:5950b99b5f66-adbfe542cf31-d6abad4bee17 \
             nix-shell \
             -I nixpkgs=./nix/nixpkgs.nix \
             --run 'python3 -c "print (\"r\n\"*1000)" | sbt << parameters.sbt_options >> << parameters.command >> '

.circleci/config/workflows/canton_build_required.yml

@@ -396,17 +396,18 @@ jobs:
         - compile # optional dependency, introduced to save credits in case of compile errors
 
 ### for release-lines we want to run data continuity tests as part of the continuous integration
-  - test_data_continuity_dumps:
-      context:
-        - datadog
-        - artifactory
-        - azure-self-cache
-        - maven-mirror
-        - da-images
-      requires:
-        - compile
-      filters:
-        branches:
-          ignore:
-            - /^gh-readonly-queue\/main\/.*/ # Do not run check on GH merge queue branches to speed up merge queue jobs
+# TODO(#33681): Disable temporarily until traffic enforcement is forward ported to main
+#  - test_data_continuity_dumps:
+#      context:
+#        - datadog
+#        - artifactory
+#        - azure-self-cache
+#        - maven-mirror
+#        - da-images
+#      requires:
+#        - compile
+#      filters:
+#        branches:
+#          ignore:
+#            - /^gh-readonly-queue\/main\/.*/ # Do not run check on GH merge queue branches to speed up merge queue jobs
 

.circleci/todo-script/src/checkTodos.sc

@@ -6,7 +6,8 @@ import scala.sys.process._
 import scala.util.matching.Regex
 import scala.util.Try
 
-/**  This script depends on `gh` (github cli) and awk
+/**  This script depends on `gh` (github cli), awk, and python3 (for the shared
+  *  exclude set in scripts/ci/todo_refs.py)
   *
   *  Current limitations of the TODO checker:
   *    - For `.rst` files, TODO blocks are considered to be the range from each instance of `todo::` to the next blank
@@ -318,44 +319,33 @@ println(s"Found ${openDamlIssues.size} open Daml issues: ${openDamlIssues.mkStri
 
 val projectRoot = "." // CI scripts are called from the project root
 
-val scalaStyleExcludeDirectories =
-  Seq(
-    "todo-script",
-    "lib",
-    "log",
-    "todo-out",
-    "theory",
-    "docs",
-    "docs-open",
-    ".git",
-    ".github",
-    ".direnv",
-    ".idea",
-    "target",
-    "3rdparty",
-    "build",
-    "contributing",
-    "daml",
+// The exclude set (directories and file globs) is owned by the shared
+// scripts/ci/todo_refs.py, so this todo checker, the issue-close workflow, and
+// the stale-flake closer cannot drift apart. `--format=grep` emits the
+// --exclude-dir/--exclude flags this grep call needs.
+val sharedExcludeArgs: Seq[String] =
+  Try("python3 scripts/ci/todo_refs.py excludes --format=grep".!!).fold(
+    err => {
+      System.err.println(
+        "Failed to obtain the shared TODO exclude set. This checker now requires python3 on " +
+          "PATH (provided by the nix shell); run it via '.ci/nix-exec amm ...'. Underlying error: " +
+          err.getMessage
+      )
+      sys.exit(1)
+    },
+    _.linesIterator.filter(_.nonEmpty).toSeq,
   )
 
+// Gate-only excludes: .github and .direnv are deliberately NOT in the shared exclude set (the
+// issue-close workflow and stale-flake closer scan .github to catch workflow TODOs). The PR gate
+// excludes them because .github/actions/build/todo_checker/action.yml has descriptive "TODO/FIXME"
+// text with no issue reference that would otherwise be flagged here as an ownerless TODO.
+val gateOnlyExcludeArgs = Seq("--exclude-dir=.github", "--exclude-dir=.direnv")
+
 // Different versions of grep (e.g. on Mac and Ubuntu) behave differently. This grep call should be tested for both
 // platforms if it is to be run locally. Right now it's only tested for Ubuntu because the CI uses Ubuntu.
-def grepForPattern(pattern: String): Seq[String] = {
-  val excludeDirectories = scalaStyleExcludeDirectories ++ Seq("release-notes")
-
-  Seq("grep", "-r") ++ excludeDirectories.map(dir => s"--exclude-dir=$dir") ++ Seq(
-    "-I", // Ignore binary files
-    "--exclude=*.png",
-    "--exclude=*checkTodos.sc",
-    "--exclude=*UNRELEASED.md",
-    "--exclude=*check_todos_on_issue_close.yml",
-    "--exclude=*close_stale_flake_issues.yml",
-    "--exclude=*close_stale_flake_issues.py",
-    "--exclude=*todo-exclude-globs.txt",
-    pattern,
-    projectRoot,
-  )
-}
+def grepForPattern(pattern: String): Seq[String] =
+  Seq("grep", "-r", "-I") ++ sharedExcludeArgs ++ gateOnlyExcludeArgs ++ Seq(pattern, projectRoot) // -I ignores binary files
 
 object ErrorCollector extends ProcessLogger {
   private val allErrors = ListBuffer[String]()

UNRELEASED.md

@@ -71,6 +71,9 @@ The Ledger API update service now exposes a `GetUpdateByHash` endpoint. Given a
   either at user creation time or through the `GrantAccessRight` command.
 - *BREAKING*: Unknown config keys are now making config parsing failing. This mechanism was already in place, but it didn't include all the config keys, which is now fixed.
 - *BREAKING*: Separated the config for support of dev and alpha protocol versions. In order to use pv=dev, you now have to specify `dev-version-support = true` instead of `alpha-version-support = true` (`canton.parameters.non-standard-config = true` is still needed).
+- The JSON Ledger API now rejects malformed identifiers (e.g. template-id) provided in requests (one that does not follow the
+  `<package>:<moduleName>:<entityName>` format) with a descriptive `400 Bad Request` error that names the
+  expected format, instead of surfacing it as an internal error.
 - The default size of the Ledger API in-memory fan-out buffer (`<participant>.ledger-api.index-service.max-transactions-in-memory-fan-out-buffer-size`) has been increased from 1000 to 1100 to accommodate serving ACS commitments from the buffer.
 - Ledger API and Indexer metrics clean-up:
   - *BREAKING*: The indexer queue metrics `daml.participant.api.indexer.indexer_queue_blocked`, `daml.participant.api.indexer.indexer_queue_buffered` and `daml.participant.api.indexer.indexer_queue_uncommitted` are now exposed as gauges instead of meters.