Run fle2v2 spec tests on modern servers; add CSFLE prose tests 1 and 11.5/11.6

spec/integration/client_side_encryption/custom_key_material_prose_spec.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'Client-Side Encryption' do
+  describe 'Prose tests: Custom Key Material Test' do
+    require_libmongocrypt
+    include_context 'define shared FLE helpers'
+
+    let(:client) do
+      ClientRegistry.instance.new_local_client(SpecConfig.instance.addresses)
+    end
+
+    let(:client_encryption) do
+      Mongo::ClientEncryption.new(
+        client,
+        key_vault_namespace: key_vault_namespace,
+        kms_providers: local_kms_providers
+      )
+    end
+
+    let(:key_vault_collection) do
+      client.use(key_vault_db)[key_vault_coll, write_concern: { w: :majority }]
+    end
+
+    # 96 bytes of custom key material, given by the spec as base64.
+    # Pass it as a String: DataKeyContext wraps it in a BSON::Binary itself.
+    let(:key_material) do
+      Base64.decode64(
+        'xPTAjBRG5JiPm+d3fj6XLi2q5DMXUS/f1f+SMAlhhwkhDRL0kr8r9GDLIGTAGlvC' \
+        '+HVjSIgdL+RKwZCvpXSyxTICWSXTUYsWYPyu3IoHbuBZdmw2faM3WhcRIgbMReU5'
+      )
+    end
+
+    # The all-zero UUID the key document is re-inserted under.
+    let(:key_id) do
+      BSON::Binary.new(Base64.decode64('AAAAAAAAAAAAAAAAAAAAAA=='), :uuid)
+    end
+
+    let(:expected_ciphertext) do
+      'AQAAAAAAAAAAAAAAAAAAAAACz0ZOLuuhEYi807ZXTdhbqhLaS2/t9wLifJnnNYwiw79d' \
+        '75QYIZ6M/aYC1h9nCzCjZ7pGUpAuNnkUhnIXM3PjrA=='
+    end
+
+    before do
+      key_vault_collection.drop
+    end
+
+    it 'encrypts with the custom key material' do
+      created_key_id = client_encryption.create_data_key(
+        'local',
+        key_material: key_material
+      )
+
+      key_document = key_vault_collection.find(_id: created_key_id).first
+      key_vault_collection.delete_one(_id: created_key_id)
+
+      key_vault_collection.insert_one(key_document.merge('_id' => key_id))
+
+      encrypted = client_encryption.encrypt(
+        'test',
+        key_id: key_id,
+        algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'
+      )
+
+      expect(Base64.strict_encode64(encrypted.data)).to eq(expected_ciphertext)
+    end
+  end
+end

spec/integration/client_side_encryption/kms_tls_options_spec.rb

@@ -428,5 +428,195 @@
 
       it_behaves_like 'it respect KMS TLS options'
     end
+
+    context 'Case 5: tlsDisableOCSPEndpointCheck is permitted' do
+      # The driver spells this option :ssl_verify_ocsp_endpoint; the URI form is
+      # tlsDisableOCSPEndpointCheck.
+      let(:client_encryption_ocsp) do
+        Mongo::ClientEncryption.new(
+          client,
+          {
+            kms_providers: {
+              aws: {
+                access_key_id: 'foo',
+                secret_access_key: 'bar'
+              }
+            },
+            kms_tls_options: {
+              aws: {
+                ssl_verify_ocsp_endpoint: false
+              }
+            },
+            key_vault_namespace: 'keyvault.datakeys',
+          }
+        )
+      end
+
+      it 'does not raise an error' do
+        expect { client_encryption_ocsp }.not_to raise_error
+      end
+    end
+
+    context 'Case 6: named KMS providers apply TLS options' do
+      let(:client_encryption_with_names) do
+        Mongo::ClientEncryption.new(
+          client,
+          {
+            kms_providers: {
+              'aws:no_client_cert' => {
+                access_key_id: SpecConfig.instance.fle_aws_key,
+                secret_access_key: SpecConfig.instance.fle_aws_secret
+              },
+              'azure:no_client_cert' => {
+                tenant_id: SpecConfig.instance.fle_azure_tenant_id,
+                client_id: SpecConfig.instance.fle_azure_client_id,
+                client_secret: SpecConfig.instance.fle_azure_client_secret,
+                identity_platform_endpoint: '127.0.0.1:8002'
+              },
+              'gcp:no_client_cert' => {
+                email: SpecConfig.instance.fle_gcp_email,
+                private_key: SpecConfig.instance.fle_gcp_private_key,
+                endpoint: '127.0.0.1:8002'
+              },
+              'kmip:no_client_cert' => {
+                endpoint: '127.0.0.1:5698'
+              },
+              'aws:with_tls' => {
+                access_key_id: SpecConfig.instance.fle_aws_key,
+                secret_access_key: SpecConfig.instance.fle_aws_secret
+              },
+              'azure:with_tls' => {
+                tenant_id: SpecConfig.instance.fle_azure_tenant_id,
+                client_id: SpecConfig.instance.fle_azure_client_id,
+                client_secret: SpecConfig.instance.fle_azure_client_secret,
+                identity_platform_endpoint: '127.0.0.1:8002'
+              },
+              'gcp:with_tls' => {
+                email: SpecConfig.instance.fle_gcp_email,
+                private_key: SpecConfig.instance.fle_gcp_private_key,
+                endpoint: '127.0.0.1:8002'
+              },
+              'kmip:with_tls' => {
+                endpoint: '127.0.0.1:5698'
+              }
+            },
+            kms_tls_options: {
+              'aws:no_client_cert' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              },
+              'azure:no_client_cert' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              },
+              'gcp:no_client_cert' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              },
+              'kmip:no_client_cert' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file
+              },
+              'aws:with_tls' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file,
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+              },
+              'azure:with_tls' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file,
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+              },
+              'gcp:with_tls' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file,
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+              },
+              'kmip:with_tls' => {
+                ssl_ca_cert: SpecConfig.instance.fle_kmip_tls_ca_file,
+                ssl_cert: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+                ssl_key: SpecConfig.instance.fle_kmip_tls_certificate_key_file,
+              }
+            },
+            key_vault_namespace: 'keyvault.datakeys',
+          }
+        )
+      end
+
+      shared_examples 'it applies named KMS TLS options' do
+        it 'TLS handshake failed without a client certificate' do
+          expect do
+            client_encryption_with_names.create_data_key(
+              "#{kms_provider}:no_client_cert",
+              { master_key: master_key }
+            )
+          end.to raise_error(Mongo::Error::KmsError, /(certificate[ _]required|SocketError|ECONNRESET)/i)
+        end
+
+        it 'TLS handshake passes with a client certificate' do
+          if expected_with_tls_error
+            expect do
+              client_encryption_with_names.create_data_key(
+                "#{kms_provider}:with_tls",
+                { master_key: master_key }
+              )
+            end.to raise_error(Mongo::Error::KmsError, expected_with_tls_error)
+          else
+            expect do
+              client_encryption_with_names.create_data_key(
+                "#{kms_provider}:with_tls",
+                { master_key: master_key }
+              )
+            end.not_to raise_error
+          end
+        end
+      end
+
+      context 'named AWS' do
+        let(:kms_provider) { 'aws' }
+
+        let(:master_key) do
+          {
+            region: 'us-east-1',
+            key: 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0',
+            endpoint: '127.0.0.1:8002'
+          }
+        end
+
+        let(:expected_with_tls_error) { /parse error/ }
+
+        it_behaves_like 'it applies named KMS TLS options'
+      end
+
+      context 'named Azure' do
+        let(:kms_provider) { 'azure' }
+
+        let(:master_key) do
+          { key_vault_endpoint: 'doesnotexist.invalid', key_name: 'foo' }
+        end
+
+        let(:expected_with_tls_error) { /HTTP status=404/ }
+
+        it_behaves_like 'it applies named KMS TLS options'
+      end
+
+      context 'named GCP' do
+        let(:kms_provider) { 'gcp' }
+
+        let(:master_key) do
+          { project_id: 'foo', location: 'bar', key_ring: 'baz', key_name: 'foo' }
+        end
+
+        let(:expected_with_tls_error) { /HTTP status=404/ }
+
+        it_behaves_like 'it applies named KMS TLS options'
+      end
+
+      context 'named KMIP' do
+        let(:kms_provider) { 'kmip' }
+
+        let(:master_key) { {} }
+
+        let(:expected_with_tls_error) { nil }
+
+        it_behaves_like 'it applies named KMS TLS options'
+      end
+    end
   end
 end

spec/spec_tests/data/client_side_encryption/fle2v2-BypassQueryAnalysis.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Compact.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-CreateCollection.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-DecryptExistingData.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Delete.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-jsonSchema.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFieldsMap-defaults.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-FindOneAndUpdate.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Indexed.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Unindexed.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-MissingKey.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-NoEncryption.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Update.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    maxServerVersion: "7.99.99"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-validatorAndPartialFieldExpression.yml

@@ -2,7 +2,6 @@
 runOn:
     # Require server version 6.0.0 to get behavior added in SERVER-64911.
     - minServerVersion: "7.0.0"
-      maxServerVersion: "7.99.99"
       # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
       # FLE 2 Encrypted collections are not supported on standalone.
       topology: [ "replicaset", "sharded", "load-balanced" ]