Support incremental builds with probe and enumeration fingerprinting

.azuredevops/pipelines/pr-ci.yml

@@ -181,6 +181,9 @@ jobs:
     parameters:
       RepoRoot: $(Build.SourcesDirectory)/MSBuildCache
       MSBuildPath: $(MSBuildPath)
+      # This job builds MSBuild from tip of main, which carries the enumeration pattern, so the
+      # scenarios must actually run here rather than skip.
+      RequireEnumerationCapability: true
 
   - publish: $(LogDirectory)
     displayName: Publish Logs

.azuredevops/pipelines/templates/e2e-test.yml

@@ -4,22 +4,34 @@ parameters:
 - name: MSBuildPath 
   type: string
   default: ""
+# Set by jobs whose MSBuild is expected to report FileAccessData.EnumeratePattern, so that a
+# missing capability fails instead of silently skipping the scenarios.
+- name: RequireEnumerationCapability
+  type: boolean
+  default: false
 
 steps:
 - task: PowerShell@2
-  displayName: "E2E Test: Microsoft.MSBuildCache.Local"
+  displayName: "E2E Smoke: Microsoft.MSBuildCache.Local"
   inputs:
-    filePath: ${{ parameters.RepoRoot }}\tests\test.ps1
+    filePath: ${{ parameters.RepoRoot }}\tests\smoke.ps1
     arguments: -MSBuildPath "${{ parameters.MSBuildPath }}" -Configuration $(BuildConfiguration) -LogDirectory "$(LogDirectory)\Tests\Local" -LocalPackageDir "$(Pipeline.Workspace)\artifacts\$(BuildConfiguration)\packages" -CachePackage Microsoft.MSBuildCache.Local
     pwsh: true
 
 - task: PowerShell@2
-  displayName: "E2E Test: Microsoft.MSBuildCache.AzurePipelines"
+  displayName: "E2E Scenarios: probe and enumeration fingerprinting"
+  inputs:
+    filePath: ${{ parameters.RepoRoot }}\tests\scenarios.ps1
+    arguments: -MSBuildPath "${{ parameters.MSBuildPath }}" -Configuration $(BuildConfiguration) -LogDirectory "$(LogDirectory)\Scenarios" -LocalPackageDir "$(Pipeline.Workspace)\artifacts\$(BuildConfiguration)\packages" -CachePackage Microsoft.MSBuildCache.Local -RequireEnumerationCapability $${{ parameters.RequireEnumerationCapability }}
+    pwsh: true
+
+- task: PowerShell@2
+  displayName: "E2E Smoke: Microsoft.MSBuildCache.AzurePipelines"
   # The access token from forks do not have enough scopes to access the pipeline cache, so skip these tests.
   # Note to repo maintainers: You can manually run the pipeline against the commit, even if the commit is from a fork, if you wish to test this.
   condition: ne(variables['System.PullRequest.IsFork'], 'True')
   inputs:
-    filePath: ${{ parameters.RepoRoot }}\tests\test.ps1
+    filePath: ${{ parameters.RepoRoot }}\tests\smoke.ps1
     arguments: -MSBuildPath "${{ parameters.MSBuildPath }}" -Configuration $(BuildConfiguration) -LogDirectory "$(LogDirectory)\Tests\AzurePipelines" -LocalPackageDir "$(Pipeline.Workspace)\artifacts\$(BuildConfiguration)\packages" -CachePackage Microsoft.MSBuildCache.AzurePipelines
     pwsh: true
   env:

CONTRIBUTING.md

@@ -56,8 +56,6 @@ Finally, add a `PackageReference` to MSBuildCache to your test repo with version
 
 **NOTE!** Because you're using a locally built package, you may need to clear it from your package cache after each iteration via a command like `rmdir /S /Q %NUGET_PACKAGES%\Microsoft.MSBuildCache` (if you set `%NUGET_PACKAGES%`) or `rmdir /S /Q %USERPROFILE%\.nuget\packages\Microsoft.MSBuildCache` if you're using the dfault package cache location. Additionally, to ensure you're not using the head version of the package, you may need to create a branch and dummy commit locally to ensure the version is higher.
 
-**NOTE!** MSBuildCache currently does not handle incremental builds well! The current target scenario is for CI environments, so **it's expected that the repo is always clean before building**. The main reason for this gap is because file probes and directory enumerations are not currently considered.
-
 To enable file reporting via detours in MSBuild, ensure `/graph` and `/reportfileaccesses` are used.
 
 Example of a set of commands to test MSBuildCache e2e in some repo:

README.md

@@ -3,9 +3,6 @@
 
 This project provides plugin implementations for the experimental [MSBuild Project Cache](https://github.com/dotnet/msbuild/blob/main/documentation/specs/project-cache.md), which enables project-level caching within MSBuild.
 
-> [!IMPORTANT]
-> Currently MSBuildCache assumes that the build is running in a clean repo. Incremental builds, e.g. local developer builds, are not supported. Target scenarios include PR builds and CI builds.
-
 ## Usage
 
 This feature requires Visual Studio 17.9 or later.
@@ -76,6 +73,7 @@ These settings are common across all plugins, although different implementations
 | `$(MSBuildCacheSkipUnchangedOutputFiles)` | `bool` | false | Whether to avoid writing output files on cache hit if the file is unchanged, which can improve performance for incremental builds. A file is considered unchanged if it exists, the previously placed file and file to be placed have the same hash, and the the previously placed file and current file on disk have the same timestamp and file size. |
 | `$(MSBuildCacheTouchOutputFiles)` | `bool` | false | Whether to update the last write time for output files on cache hit. All files for a given cache entry will have the same timestamp. Note that outputs which skip materialization via `MSBuildCacheSkipUnchangedOutputFiles` are still touched. |
 | `$(MSBuildCacheIgnoreDotNetSdkPatchVersion)` | `bool` | false | Whether to ignore the patch version when doing cache lookups. This trades off some correctness for the sake of getting cache hits when the SDK version isn't exactly the same. The default behavior is to consider the exact SDK version, eg. "8.0.404". With this setting set to true, it will instead use something like "8.0.4XX". Note that the major version, minor version, and feature bands are still considered. |
+| `$(MSBuildCacheEnableProbeAndEnumerationFingerprinting)` | `bool` | true | Whether file probes (existence checks) and directory enumerations contribute to the strong fingerprint, enabling correct caching for incremental builds — including cases where MSBuild source globs match different files. Requires an MSBuild that reports directory enumeration patterns; on older versions this is forced to `false` and a message is logged. |
 
 When configuring settings which are list types, you should always append to the existing value to avoid overriding the defaults:
 

src/Common.Tests/ByRefGetterFactoryTests.cs

@@ -0,0 +1,135 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.MSBuildCache.FileAccess;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.MSBuildCache.Tests;
+
+/// <summary>
+/// Covers binding against both shapes of MSBuild's <c>FileAccessData</c>: the older one that lacks
+/// the enumeration fields and the newer one that carries them. The real struct comes from whichever
+/// <c>Microsoft.Build.dll</c> the host supplies, so stand-in structs are used to exercise both sides
+/// without needing two MSBuild installations.
+/// </summary>
+[TestClass]
+public sealed class ByRefGetterFactoryTests
+{
+    private enum StandInAttributes : uint
+    {
+        None = 0,
+        Directory = 0x10,
+    }
+
+    /// <summary>Mirrors the pre-18.10 FileAccessData: no enumeration fields.</summary>
+    private struct OlderFileAccessData
+    {
+        private string _path;
+
+        public OlderFileAccessData(string path) => _path = path;
+
+        public string Path
+        {
+            readonly get => _path;
+            private set => _path = value;
+        }
+    }
+
+    /// <summary>Mirrors the newer FileAccessData, including the readonly-get/private-set shape.</summary>
+    private struct NewerFileAccessData
+    {
+        private string _path;
+        private string? _enumeratePattern;
+        private StandInAttributes _openedAttributes;
+
+        public NewerFileAccessData(string path, string? enumeratePattern, StandInAttributes openedAttributes)
+        {
+            _path = path;
+            _enumeratePattern = enumeratePattern;
+            _openedAttributes = openedAttributes;
+        }
+
+        public string Path
+        {
+            readonly get => _path;
+            private set => _path = value;
+        }
+
+        public string? EnumeratePattern
+        {
+            readonly get => _enumeratePattern;
+            private set => _enumeratePattern = value;
+        }
+
+        public StandInAttributes OpenedFileOrDirectoryAttributes
+        {
+            readonly get => _openedAttributes;
+            private set => _openedAttributes = value;
+        }
+    }
+
+    [TestMethod]
+    public void ReturnsNullWhenPropertyIsAbsent()
+    {
+        Assert.IsNull(ByRefGetterFactory.TryCreate<OlderFileAccessData, string?>("EnumeratePattern"));
+        Assert.IsNull(ByRefGetterFactory.TryCreate<OlderFileAccessData, StandInAttributes>("OpenedFileOrDirectoryAttributes"));
+    }
+
+    [TestMethod]
+    public void ReturnsNullWhenPropertyTypeDiffers()
+    {
+        // Guards against silently binding to a property that was reshaped rather than added.
+        Assert.IsNull(ByRefGetterFactory.TryCreate<NewerFileAccessData, int>("EnumeratePattern"));
+    }
+
+    [TestMethod]
+    public void ReadsStringPropertyWithoutBoxing()
+    {
+        ByRefGetter<NewerFileAccessData, string?>? getter =
+            ByRefGetterFactory.TryCreate<NewerFileAccessData, string?>("EnumeratePattern");
+        Assert.IsNotNull(getter);
+
+        NewerFileAccessData data = new(@"X:\dir", "*.cs", StandInAttributes.Directory);
+        Assert.AreEqual("*.cs", getter(ref data));
+    }
+
+    [TestMethod]
+    public void ReadsEnumProperty()
+    {
+        ByRefGetter<NewerFileAccessData, StandInAttributes>? getter =
+            ByRefGetterFactory.TryCreate<NewerFileAccessData, StandInAttributes>("OpenedFileOrDirectoryAttributes");
+        Assert.IsNotNull(getter);
+
+        NewerFileAccessData data = new(@"X:\dir", "*.cs", StandInAttributes.Directory);
+        Assert.AreEqual(StandInAttributes.Directory, getter(ref data));
+    }
+
+    [TestMethod]
+    public void ReadsNullStringProperty()
+    {
+        ByRefGetter<NewerFileAccessData, string?>? getter =
+            ByRefGetterFactory.TryCreate<NewerFileAccessData, string?>("EnumeratePattern");
+        Assert.IsNotNull(getter);
+
+        NewerFileAccessData data = new(@"X:\dir", enumeratePattern: null, StandInAttributes.None);
+        Assert.IsNull(getter(ref data));
+    }
+
+    /// <summary>
+    /// The getter is bound once and reused across every reported file access, so it must observe the
+    /// instance it is handed rather than a snapshot captured at bind time.
+    /// </summary>
+    [TestMethod]
+    public void BoundGetterIsReusableAcrossInstances()
+    {
+        ByRefGetter<NewerFileAccessData, string?>? getter =
+            ByRefGetterFactory.TryCreate<NewerFileAccessData, string?>("EnumeratePattern");
+        Assert.IsNotNull(getter);
+
+        NewerFileAccessData first = new(@"X:\a", "*.cs", StandInAttributes.None);
+        NewerFileAccessData second = new(@"X:\b", "*.dll", StandInAttributes.None);
+
+        Assert.AreEqual("*.cs", getter(ref first));
+        Assert.AreEqual("*.dll", getter(ref second));
+    }
+}