Tool to investigate SQLite compression

.github/actions/spelling/expect.txt

@@ -169,6 +169,7 @@ ewgs
 execustom
 EXEHASH
 experimentalfeatures
+XPRESS
 fdw
 fdwgp
 FECAFEB

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp

@@ -8,10 +8,18 @@
 
 namespace AppInstaller::Repository::Microsoft
 {
+    namespace
+    {
+        size_t GetPageSizeFromOptions(SQLiteIndex::CreateOptions options)
+        {
+            return WI_IsFlagSet(options, SQLiteIndex::CreateOptions::LargePageSize) ? 65536 : 0;
+        }
+    }
+
     SQLiteIndex SQLiteIndex::CreateNew(const std::string& filePath, SQLite::Version version, CreateOptions options)
     {
         AICLI_LOG(Repo, Info, << "Creating new SQLite Index with version [" << version << "] at '" << filePath << "'");
-        SQLiteIndex result{ filePath, version };
+        SQLiteIndex result{ filePath, version, options };
 
         SQLite::Savepoint savepoint = SQLite::Savepoint::Create(result.m_dbconn, "sqliteindex_createnew");
 
@@ -37,7 +45,7 @@ namespace AppInstaller::Repository::Microsoft
         return { filePath, source };
     }
 
-    SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version) : SQLiteStorageBase(target, version)
+    SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options))
     {
         m_dbconn.EnableICU();
         m_interface = Schema::CreateISQLiteIndex(version);

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h

@@ -175,7 +175,7 @@ namespace AppInstaller::Repository::Microsoft
 
     private:
         // Constructor used to create a new index.
-        SQLiteIndex(const std::string& target, const SQLite::Version& version);
+        SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options);
 
         // Constructor used to open an existing index.
         SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile);

src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h

@@ -52,6 +52,11 @@ namespace AppInstaller::Repository::Microsoft::Schema
             SupportPathless = 0x1,
             // Disable support for dependencies
             DisableDependenciesSupport = 0x2,
+            // Use maximum page size in SQLite.
+            // This was part of an exploration of ways to reduce file size but ultimately led to a larger
+            // compressed file with a worse ratio (limited testing but was significant enough to warrant abandonment).
+            // Leaving this here as a valid null result to prevent future maintainers from needing to investigate.
+            LargePageSize = 0x4,
         };
 
         // Contains both the object representation of the version key and the rows.

src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h

@@ -39,7 +39,7 @@ namespace AppInstaller::SQLite
         static void RenameSQLiteDatabase(const std::filesystem::path& source, const std::filesystem::path& destination, bool overwrite = false);
 
     protected:
-        SQLiteStorageBase(const std::string& target, const Version& version);
+        SQLiteStorageBase(const std::string& target, const Version& version, size_t pageSize = 0);
 
         SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile);
 

src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h

@@ -264,6 +264,11 @@ namespace AppInstaller::SQLite
         // Must be performed outside of a transaction.
         bool SetJournalMode(std::string_view mode);
 
+        // Sets the page size for a new, empty database.
+        // Must be called before the first write to take effect.
+        // Must be a power of two between 512 and 65536 (inclusive), but we let SQLite enforce that.
+        void SetPageSize(size_t pageSize);
+
         operator sqlite3* () const { return m_dbconn->Get(); }
 
     protected:

src/AppInstallerSharedLib/SQLiteStorageBase.cpp

@@ -156,10 +156,14 @@ namespace AppInstaller::SQLite
         m_version = Version::GetSchemaVersion(m_dbconn);
     }
 
-    SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Version& version) :
+    SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Version& version, size_t pageSize) :
         m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create))
     {
         m_version = version;
+        if (pageSize > 0)
+        {
+            m_dbconn.SetPageSize(pageSize);
+        }
         MetadataTable::Create(m_dbconn);
 
         // Write a new identifier for this database

src/AppInstallerSharedLib/SQLiteWrapper.cpp

@@ -251,6 +251,15 @@ namespace AppInstaller::SQLite
         return ToLower(setJournalMode.GetColumn<std::string>(0)) == ToLower(mode);
     }
 
+    void Connection::SetPageSize(size_t pageSize)
+    {
+        std::ostringstream stream;
+        stream << "PRAGMA page_size=" << pageSize;
+
+        Statement setPageSize = Statement::Create(*this, stream.str());
+        setPageSize.Step();
+    }
+
     std::shared_ptr<details::SharedConnection> Connection::GetSharedConnection() const
     {
         return m_dbconn;

tools/IndexComparisonTool/IndexComparisonTool.vcxproj

@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="16.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+
+  <PropertyGroup Label="Globals">
+    <VCProjectVersion>16.0</VCProjectVersion>
+    <ProjectGuid>{2F8A1C3E-7B4D-4E9F-A6C2-1D5E8B3F0A7C}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>IndexComparisonTool</RootNamespace>
+    <WindowsTargetPlatformVersion>10.0.26100.0</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformMinVersion>10.0.17763.0</WindowsTargetPlatformMinVersion>
+    <WindowsSDKDesktopARM64Support>true</WindowsSDKDesktopARM64Support>
+  </PropertyGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
+            Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')"
+            Label="LocalAppDataPlatform" />
+  </ImportGroup>
+
+  <PropertyGroup Label="UserMacros" />
+
+  <PropertyGroup>
+    <OutDir>$(ProjectDir)bin\$(Platform)\$(Configuration)\</OutDir>
+    <IntDir>$(ProjectDir)obj\$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PrecompiledHeader>Use</PrecompiledHeader>
+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+      <PrecompiledHeaderOutputFile>$(IntDir)pch.pch</PrecompiledHeaderOutputFile>
+      <WarningLevel>Level4</WarningLevel>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <SDLCheck>true</SDLCheck>
+      <AdditionalOptions>/permissive- /std:c++17 %(AdditionalOptions)</AdditionalOptions>
+      <!-- WinGetUtil.h is local to this project; no extra include dir needed -->
+      <PreprocessorDefinitions>UNICODE;_UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <!-- Cabinet.lib and winsqlite3.lib are pulled in via #pragma comment(lib) in main.cpp -->
+      <!-- WinGetUtil.dll is loaded at runtime via LoadLibrary; no import lib needed -->
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <ClInclude Include="pch.h" />
+    <ClInclude Include="WinGetUtil.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="main.cpp" />
+    <ClCompile Include="pch.cpp">
+      <PrecompiledHeader>Create</PrecompiledHeader>
+    </ClCompile>
+  </ItemGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+</Project>

tools/IndexComparisonTool/IndexComparisonTool.vcxproj.filters

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hh;hpp;hxx;hm;inl;inc;ipp;xsd</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="main.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="pch.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="pch.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="WinGetUtil.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>

tools/IndexComparisonTool/WinGetUtil.h

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// Minimal WinGetUtil types and function pointer declarations for use
+// with runtime loading via LoadLibrary. Only the APIs used by
+// IndexComparisonTool are defined here.
+#pragma once
+
+typedef void*           WINGET_SQLITE_INDEX_HANDLE;
+typedef wchar_t const*  WINGET_STRING;
+
+#define WINGET_SQLITE_INDEX_VERSION_LATEST ((UINT32)-1)
+
+typedef HRESULT (__stdcall *PFN_WinGetSQLiteIndexCreate)(
+    WINGET_STRING               filePath,
+    UINT32                      majorVersion,
+    UINT32                      minorVersion,
+    WINGET_SQLITE_INDEX_HANDLE* index);
+
+typedef HRESULT (__stdcall *PFN_WinGetSQLiteIndexAddManifest)(
+    WINGET_SQLITE_INDEX_HANDLE  index,
+    WINGET_STRING               manifestPath,
+    WINGET_STRING               relativePath);
+
+typedef HRESULT (__stdcall *PFN_WinGetSQLiteIndexPrepareForPackaging)(
+    WINGET_SQLITE_INDEX_HANDLE  index);
+
+typedef HRESULT (__stdcall *PFN_WinGetSQLiteIndexClose)(
+    WINGET_SQLITE_INDEX_HANDLE  index);