Delta client

doc/Settings.md

@@ -465,3 +465,15 @@ Enables numbered choices for ambiguous single-package `install`, `show`, and `do
 ```
 
 `--disable-interactivity` and redirected input or output still prevent prompting.
+
+### deltaIndex
+
+This feature changes how the client acquires updates to a pre-indexed source. Instead of downloading the whole index every time, it downloads a delta describing only what has changed since a periodically published baseline, and combines the two locally. The baseline is downloaded only when the source does not already have the one the delta was built against.
+
+The data presented by the source is the same either way; only how it is acquired differs. A source that does not publish a delta is unaffected, and the client falls back to downloading the full index.
+
+```json
+  "experimentalFeatures": {
+        "deltaIndex": true
+  },
+```

schemas/JSON/settings/settings.schema.0.2.json

@@ -348,6 +348,11 @@
           "description": "Enable support for some commands to resume",
           "type": "boolean",
           "default": false
+        },
+        "deltaIndex": {
+          "description": "Enable acquiring source index updates as a delta against a baseline, rather than downloading the full index",
+          "type": "boolean",
+          "default": false
         }
       }
     },

src/AppInstallerCLIE2ETests/Constants.cs

@@ -53,6 +53,27 @@ public class Constants
         public const string RestTestSourceUrl = @"https://localhost:5001/TestKit/TestData/TestRestSource";
         public const string RestTestSourceType = "Microsoft.Rest";
 
+        // The delta capable test source. It is published separately from the source above so that a
+        // client that knows nothing about deltas keeps seeing exactly what it saw before.
+        public const string DeltaTestSourceName = @"TestDeltaSource";
+        public const string DeltaTestSourceDirectory = @"deltaSource";
+        public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory;
+        public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests";
+        public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta";
+        public const string DeltaTestSourceIdentifier = DeltaTestSourceIdentityName + "_" + AICLIPackagePublisherHash;
+        public const string DeltaTestSourceDeltaIdentifier = DeltaTestSourceDeltaIdentityName + "_" + AICLIPackagePublisherHash;
+
+        // The baseline is published older than the full index, so that a client that already holds
+        // the full index has to move backward to pick the baseline up.
+        public const string DeltaTestSourceVersion = @"2020.805.713.335";
+        public const string DeltaTestSourceBaselineVersion = @"2020.805.713.334";
+        public const string DeltaTestSourceBaselineRelativePath = @"baseline/source2.msix";
+
+        // Left out of the baseline, so that it exists only in the delta. Finding it is what tells a
+        // merged result apart from either half read on its own.
+        public const string DeltaOnlyManifestFileName = @"TestExampleInstaller.yaml";
+        public const string DeltaOnlyPackageIdentifier = @"AppInstallerTest.TestExampleInstaller";
+
         public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe";
         public const string AICLIPackageName = "WinGetDevCLI";
         public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe";

src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs

@@ -0,0 +1,196 @@
+// -----------------------------------------------------------------------------
+// <copyright file="DeltaSourceCommand.cs" company="Microsoft Corporation">
+//     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+// </copyright>
+// -----------------------------------------------------------------------------
+
+namespace AppInstallerCLIE2ETests
+{
+    using AppInstallerCLIE2ETests.Helpers;
+    using NUnit.Framework;
+
+    /// <summary>
+    /// Tests the delta index against the deployed package mechanism.
+    ///
+    /// Unit tests cover the shape of a delta source and the merging of a delta with its baseline,
+    /// but they run unpackaged, so they can only ever reach the local file store. Everything here
+    /// is about the half they structurally cannot touch: deploying the packages, the full index and
+    /// the baseline sharing a single deployed identity, and the clean up that follows a removal.
+    /// </summary>
+    public class DeltaSourceCommand : BaseCommand
+    {
+        /// <summary>
+        /// The experimental feature, as it is named in the settings file.
+        /// </summary>
+        private const string DeltaFeatureName = "deltaIndex";
+
+        /// <summary>
+        /// A package that is in the baseline, and so is not one the delta has to supply.
+        /// </summary>
+        private const string BaselinePackageIdentifier = Constants.PortableExePackageId;
+
+        /// <summary>
+        /// One time set up.
+        /// </summary>
+        [OneTimeSetUp]
+        public void OneTimeSetup()
+        {
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+        }
+
+        /// <summary>
+        /// One time tear down.
+        /// </summary>
+        [OneTimeTearDown]
+        public void OneTimeTeardown()
+        {
+            RemoveDeltaSource();
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false);
+        }
+
+        /// <summary>
+        /// Test set up.
+        /// </summary>
+        [SetUp]
+        public void Setup()
+        {
+            RemoveDeltaSource();
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+        }
+
+        /// <summary>
+        /// Adding a delta capable source acquires the delta and the baseline rather than the full
+        /// index, and what it then presents is the two of them merged.
+        /// </summary>
+        [Test]
+        public void DeltaSourceAdd()
+        {
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed");
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceBaselineVersion),
+                "The baseline should have been deployed, not the full index");
+
+            // This package was left out of the baseline, so it exists only in the delta.
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+
+            // And a package that is in the baseline is still there, so the delta did not displace it.
+            RequireFound(BaselinePackageIdentifier);
+        }
+
+        /// <summary>
+        /// A source added while the feature was off holds the full index. Turning the feature on and
+        /// updating has to move that deployed identity backward to the baseline's older version.
+        /// </summary>
+        [Test]
+        public void DeltaSourceAdoptedByExistingFullIndexSource()
+        {
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false);
+
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be involved while the feature is off");
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceVersion),
+                "The full index should be deployed at its own version");
+
+            WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true);
+
+            var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(updateResult.StdOut, Does.Contain("Done"));
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed after the update");
+
+            // The full index and the baseline share a deployed identity, so this is the baseline
+            // having replaced a package that was newer than itself.
+            Assert.That(
+                GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName),
+                Is.EqualTo(Constants.DeltaTestSourceBaselineVersion),
+                "The baseline should have replaced the full index, even though it is older");
+
+            RequireFound(Constants.DeltaOnlyPackageIdentifier);
+        }
+
+        /// <summary>
+        /// Removing the source takes both deployed packages with it. A delta source leaves more
+        /// behind than an ordinary one, so there is more to get wrong.
+        /// </summary>
+        [Test]
+        public void DeltaSourceRemove()
+        {
+            AddDeltaSource();
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True);
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.True);
+
+            var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(removeResult.StdOut, Does.Contain("Done"));
+
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The delta package should have been removed");
+            Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False, "The baseline package should have been removed");
+
+            var listResult = TestCommon.RunAICLICommand("source list", $"-n {Constants.DeltaTestSourceName}");
+            Assert.That(listResult.ExitCode, Is.EqualTo(Constants.ErrorCode.ERROR_SOURCE_NAME_DOES_NOT_EXIST));
+        }
+
+        /// <summary>
+        /// Adds the delta source, requiring that it succeed.
+        /// </summary>
+        private static void AddDeltaSource()
+        {
+            var result = TestCommon.RunAICLICommand("source add", $"{Constants.DeltaTestSourceName} {Constants.DeltaTestSourceUrl} --trust-level trusted");
+            Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK));
+            Assert.That(result.StdOut, Does.Contain("Done"));
+        }
+
+        /// <summary>
+        /// Removes the delta source, tolerating its not having been added.
+        /// </summary>
+        private static void RemoveDeltaSource()
+        {
+            TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}");
+        }
+
+        /// <summary>
+        /// Requires that the delta source presents the given package.
+        /// </summary>
+        /// <param name="packageIdentifier">The package identifier.</param>
+        private static void RequireFound(string packageIdentifier)
+        {
+            var result = TestCommon.RunAICLICommand("search", $"{packageIdentifier} --source {Constants.DeltaTestSourceName}");
+            Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK), $"{packageIdentifier} should be found");
+            Assert.That(result.StdOut, Does.Contain(packageIdentifier));
+        }
+
+        /// <summary>
+        /// Whether a package with the given identity name is deployed for the current user.
+        /// </summary>
+        /// <param name="identityName">The package identity name.</param>
+        /// <returns>True if it is deployed.</returns>
+        private static bool IsPackageDeployed(string identityName)
+        {
+            return !string.IsNullOrEmpty(GetDeployedPackageVersion(identityName));
+        }
+
+        /// <summary>
+        /// Gets the version of a deployed package, or an empty string when it is not deployed.
+        /// </summary>
+        /// <param name="identityName">The package identity name.</param>
+        /// <returns>The version, or an empty string.</returns>
+        private static string GetDeployedPackageVersion(string identityName)
+        {
+            // The name has to match exactly, which matters here because the delta's identity name
+            // has the full index's as a prefix.
+            var result = TestCommon.RunCommandWithResult(
+                "powershell",
+                $"-Command \"(Get-AppxPackage | Where-Object {{ $_.Name -ceq '{identityName}' }} | Select-Object -First 1).Version\"");
+
+            return result.StdOut.Trim();
+        }
+    }
+}

src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs

@@ -192,6 +192,19 @@ public static void GenerateE2ESource()
                 {
                     CertFile = testParams.PackageCertificatePath,
                 },
+                Delta = new ()
+                {
+                    RelativeDirectory = Constants.DeltaTestSourceDirectory,
+                    IdentityName = Constants.DeltaTestSourceIdentityName,
+                    DeltaIdentityName = Constants.DeltaTestSourceDeltaIdentityName,
+                    Version = Constants.DeltaTestSourceVersion,
+                    BaselineVersion = Constants.DeltaTestSourceBaselineVersion,
+                    BaselineRelativePath = Constants.DeltaTestSourceBaselineRelativePath,
+                    ManifestsNotInBaseline = new ()
+                    {
+                        Constants.DeltaOnlyManifestFileName,
+                    },
+                },
             };
 
             WinGetLocalSource.CreateLocalSource(e2eSource);

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj

@@ -218,6 +218,7 @@
     <ClInclude Include="TestConfiguration.h" />
     <ClInclude Include="TestRestRequestHandler.h" />
     <ClInclude Include="TestHooks.h" />
+    <ClInclude Include="TestMsixPackage.h" />
     <ClInclude Include="TestSettings.h" />
     <ClInclude Include="TestSource.h" />
     <ClInclude Include="WorkflowCommon.h" />
@@ -338,6 +339,7 @@
     <ClCompile Include="TableOutput.cpp" />
     <ClCompile Include="TestCertificates.cpp" />
     <ClCompile Include="TestCommon.cpp" />
+    <ClCompile Include="TestMsixPackage.cpp" />
     <ClCompile Include="WorkflowCommon.cpp" />
     <ClCompile Include="WorkflowGroupPolicy.cpp" />
     <ClCompile Include="Yaml.cpp" />

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters

@@ -81,6 +81,9 @@
     <ClInclude Include="TestHooks.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="TestMsixPackage.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="TestSource.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -116,6 +119,9 @@
     <ClCompile Include="TestCommon.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="TestMsixPackage.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="TestSource.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -425,15 +431,15 @@
     <ClCompile Include="MatchCriteriaResolver.cpp">
       <Filter>Source Files\Repository</Filter>
     </ClCompile>
-    <ClCompile Include="RepairFlow.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="SQLiteIndexTestCommon.cpp">
       <Filter>Source Files\Repository</Filter>
     </ClCompile>
     <ClCompile Include="SQLiteIndexDelta.cpp">
       <Filter>Source Files\Repository</Filter>
     </ClCompile>
+    <ClCompile Include="RepairFlow.cpp">
+      <Filter>Source Files\CLI</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />

src/AppInstallerCLITests/ExperimentalFeature.cpp

@@ -28,6 +28,48 @@ TEST_CASE("ExperimentalFeature None", "[experimentalFeature]")
     REQUIRE(ExperimentalFeature::IsEnabled(ExperimentalFeature::Feature::None));
 }
 
+// Registering a feature takes four separate edits in three files, and nothing connects them. A
+// flag added to the enum without a GetFeature case makes `winget features` throw rather than
+// merely omit the feature, and one without a setting throws from IsEnabled. Both are caught here
+// for every feature at once, so the cost of the split registration is paid by this test rather
+// than by whoever adds the next one.
+TEST_CASE("ExperimentalFeature AllFeaturesAreRegistered", "[experimentalFeature]")
+{
+    auto again = DeleteUserSettingsFiles();
+
+    std::set<std::string_view> jsonNames;
+
+    for (const auto& feature : ExperimentalFeature::GetAllFeatures())
+    {
+        INFO(feature.Name());
+
+        REQUIRE(!feature.Name().empty());
+        REQUIRE(!feature.JsonName().empty());
+
+        // The json name is what a user types, so two features cannot share one.
+        REQUIRE(jsonNames.insert(feature.JsonName()).second);
+
+        // An experimental feature is off until it is asked for. The empty settings are written
+        // explicitly so that this does not depend on what a previous iteration left behind.
+        {
+            SetSetting(Stream::PrimaryUserSettings, "{}");
+            UserSettingsTest userSettingTest;
+
+            REQUIRE_FALSE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest));
+        }
+
+        // The name that `winget features` prints has to be the one that actually enables the
+        // feature, which is the only thing tying the enum value to its setting.
+        {
+            std::string json = R"({ "experimentalFeatures": { ")" + std::string{ feature.JsonName() } + R"(": true } })";
+            SetSetting(Stream::PrimaryUserSettings, json);
+            UserSettingsTest userSettingTest;
+
+            REQUIRE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest));
+        }
+    }
+}
+
 TEST_CASE("ExperimentalFeature ExperimentalCmd", "[experimentalFeature]")
 {
     auto again = DeleteUserSettingsFiles();
@@ -104,4 +146,4 @@ TEST_CASE("ExperimentalFeature InteractivePackageSelection", "[experimentalFeatu
     {
         return item.GetFeature() == ExperimentalFeature::Feature::InteractivePackageSelection;
     }));
-}
+}