Change tool metadata file format to JSON

azure-pipelines/end-to-end-tests-dir/fetch.ps1

@@ -6,90 +6,107 @@ if (-not $IsMacOS -and -not $IsLinux) {
     $Scripts = Join-Path $TestingRoot "scripts"
     mkdir $Scripts | Out-Null
 
-@"
-<?xml version="1.0"?>
-<tools version="2">
-    <tool name="7zip" os="windows">
-        <version>19.00</version>
-        <exeRelativePath>Files\7-Zip\7z.exe</exeRelativePath>
-        <url>https://www.7-zip.org/a/7z1900-x64.msi</url>
-        <sha512>7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8</sha512>
-        <archiveName>7z1900-x64.msi</archiveName>
-    </tool>
-    <tool name="ninja-testing" os="windows">
-        <version>1.10.2</version>
-        <exeRelativePath>ninja.exe</exeRelativePath>
-        <url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
-        <sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
-        <archiveName>ninja-win-1.10.2.zip</archiveName>
-    </tool>
-    <tool name="ninja" os="windows">
-        <version>1.10.2</version>
-        <exeRelativePath>ninja.exe</exeRelativePath>
-        <url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
-        <sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
-        <archiveName>ninja-win-1.10.2.zip</archiveName>
-    </tool>
-    <tool name="cmake" os="windows">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-windows-i386\bin\cmake.exe</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip</url>
-        <sha512>969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79</sha512>
-        <archiveName>cmake-3.22.2-windows-i386.zip</archiveName>
-    </tool>
-    <tool name="cmake" os="osx">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz</url>
-        <sha512>08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac</sha512>
-        <archiveName>cmake-3.22.2-macos-universal.tar.gz</archiveName>
-    </tool>
-    <tool name="cmake" os="linux">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-linux-x86_64/bin/cmake</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz</url>
-        <sha512>579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b</sha512>
-        <archiveName>cmake-3.22.2linux-x86_64.tar.gz</archiveName>
-    </tool>
-    <tool name="cmake" os="freebsd">
-        <version>3.20.4</version>
-        <exeRelativePath>usr/local/bin/cmake</exeRelativePath>
-        <url>https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz</url>
-        <sha512>3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475</sha512>
-        <archiveName>cmake-3.20.4.txz</archiveName>
-    </tool>
-</tools>
-"@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkgTools.xml")
+    $7zip_version = "19.00"
+    $ninja_version = "1.10.2"
+
+    @'
+{
+    "schema-version": 1,
+    "tools": [{
+        "name": "7zip",
+        "os": "windows",
+        "version": "19.00",
+        "executable": "Files\\7-Zip\\7z.exe",
+        "url": "https://www.7-zip.org/a/7z1900-x64.msi",
+        "sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8",
+        "archive": "7z1900-x64.msi"
+    },
+    {
+        "name": "ninja-testing",
+        "os": "windows",
+        "version": "1.10.2",
+        "executable": "ninja.exe",
+        "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+        "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+        "archive": "ninja-win-1.10.2.zip"
+    },
+    {
+        "name": "ninja",
+        "os": "windows",
+        "version": "1.10.2",
+        "executable": "ninja.exe",
+        "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+        "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+        "archive": "ninja-win-1.10.2.zip"
+    },
+    {
+        "name": "cmake",
+        "os": "windows",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip",
+        "sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79",
+        "archive": "cmake-3.22.2-windows-i386.zip"
+    },
+    {
+        "name": "cmake",
+        "os": "osx",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz",
+        "sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac",
+        "archive": "cmake-3.22.2-macos-universal.tar.gz"
+    },
+    {
+        "name": "cmake",
+        "os": "linux",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-linux-x86_64/bin/cmake",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz",
+        "sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b",
+        "archive": "cmake-3.22.2linux-x86_64.tar.gz"
+    },
+    {
+        "name": "cmake",
+        "os": "freebsd",
+        "version": "3.20.4",
+        "executable": "/usr/local/bin/cmake",
+        "url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz",
+        "sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475",
+        "archive": "cmake-3.20.4.txz"
+    }]
+}
+'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkg-tools.json")
 
     $env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads'
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
 
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe"
 
     $path = $env:PATH
 
-    $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows"
+    $env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
-    Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
+    Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
     Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES
 
     $env:VCPKG_FORCE_SYSTEM_BINARIES = "1"
     $env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES
     $out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status"))

include/vcpkg/base/contractual-constants.h

@@ -290,6 +290,7 @@ namespace vcpkg
     inline constexpr StringLiteral SwitchTargetX86 = "target:x86";
     inline constexpr StringLiteral SwitchTLogFile = "tlog-file";
     inline constexpr StringLiteral SwitchTools = "tools";
+    inline constexpr StringLiteral SwitchToolDataFile = "tool-data-file";
     inline constexpr StringLiteral SwitchTriplet = "triplet";
     inline constexpr StringLiteral SwitchUrl = "url";
     inline constexpr StringLiteral SwitchVcpkgRoot = "vcpkg-root";

include/vcpkg/base/jsonreader.h

@@ -361,4 +361,20 @@ namespace vcpkg::Json
         virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
         static const FeatureNameDeserializer instance;
     };
+
+    struct ArchitectureDeserializer final : Json::IDeserializer<std::string>
+    {
+        static const std::vector<StringLiteral> KNOWN_ARCHITECTURES;
+
+        virtual LocalizedString type_name() const override;
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
+        static const ArchitectureDeserializer instance;
+    };
+
+    struct Sha512Deserializer final : Json::IDeserializer<std::string>
+    {
+        virtual LocalizedString type_name() const override;
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
+        static const Sha512Deserializer instance;
+    };
 }

include/vcpkg/base/message-data.inc.h

@@ -4,13 +4,15 @@ DECLARE_MESSAGE(ADefaultFeature, (), "", "a default feature")
 DECLARE_MESSAGE(ABoolean, (), "", "a boolean")
 DECLARE_MESSAGE(ABuiltinRegistry, (), "", "a builtin registry")
 DECLARE_MESSAGE(AConfigurationObject, (), "", "a configuration object")
+DECLARE_MESSAGE(ACpuArchitecture, (), "", "a CPU architecture")
 DECLARE_MESSAGE(ADependency, (), "", "a dependency")
 DECLARE_MESSAGE(ADependencyFeature, (), "", "a feature of a dependency")
 DECLARE_MESSAGE(ADemandObject,
                 (),
                 "'demands' are a concept in the schema of a JSON file the user can edit",
                 "a demand object")
 DECLARE_MESSAGE(AString, (), "", "a string")
+DECLARE_MESSAGE(ASha512, (), "", "a SHA-512 hash")
 DECLARE_MESSAGE(ADateVersionString, (), "", "a date version string")
 DECLARE_MESSAGE(AddArtifactOnlyOne, (msg::command_line), "", "'{command_line}' can only add one artifact at a time.")
 DECLARE_MESSAGE(AddCommandFirstArg, (), "", "The first parameter to add must be 'artifact' or 'port'.")
@@ -264,6 +266,13 @@ DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string")
 DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features")
 DECLARE_MESSAGE(AStringOrArrayOfStrings, (), "", "a string or array of strings")
 DECLARE_MESSAGE(AStringStringDictionary, (), "", "a \"string\": \"string\" dictionary")
+DECLARE_MESSAGE(AToolDataObject, (), "", "tool metadata")
+DECLARE_MESSAGE(AToolDataArray, (), "", "an array of tool metadata")
+DECLARE_MESSAGE(AToolDataFile, (), "", "a tool data file")
+DECLARE_MESSAGE(ToolDataFileSchemaVersionNotSupported,
+                (msg::version),
+                "",
+                "document schema version {version} is not supported by this version of vcpkg")
 DECLARE_MESSAGE(AttemptingToSetBuiltInBaseline,
                 (),
                 "",
@@ -929,10 +938,6 @@ DECLARE_MESSAGE(CouldNotFindGitTreeAtCommit,
                 (msg::package_name, msg::commit_sha),
                 "",
                 "could not find the git tree for `versions` in repo {package_name} at commit {commit_sha}")
-DECLARE_MESSAGE(CouldNotFindToolVersion,
-                (msg::version, msg::path),
-                "",
-                "Could not find <tools version=\"{version}\"> in {path}")
 DECLARE_MESSAGE(CouldNotFindVersionDatabaseFile, (msg::path), "", "Couldn't find the versions database file: {path}")
 DECLARE_MESSAGE(CreatedNuGetPackage, (msg::path), "", "Created nupkg: {path}")
 DECLARE_MESSAGE(CreateFailureLogsDir, (msg::path), "", "Creating failure logs output directory {path}.")
@@ -1168,6 +1173,10 @@ DECLARE_MESSAGE(ErrorWhileFetchingBaseline,
                 "{value} is a commit sha.",
                 "while fetching baseline `\"{value}\"` from repo {package_name}:")
 DECLARE_MESSAGE(ErrorWhileParsing, (msg::path), "", "Errors occurred while parsing {path}.")
+DECLARE_MESSAGE(ErrorWhileParsingToolData,
+                (msg::path),
+                "",
+                "An unexpected error ocurred while parsing tool data from {path}.")
 DECLARE_MESSAGE(ErrorWhileWriting, (msg::path), "", "Error occurred while writing {path}.")
 DECLARE_MESSAGE(ExamplesHeader, (), "Printed before a list of example command lines", "Examples:")
 DECLARE_MESSAGE(ExceededRecursionDepth, (), "", "Recursion depth exceeded.")
@@ -1832,6 +1841,11 @@ DECLARE_MESSAGE(InvalidArchitecture,
                 (msg::value),
                 "{value} is what the user entered that we did not understand",
                 "invalid architecture: {value}")
+DECLARE_MESSAGE(
+    InvalidArchitectureValue,
+    (msg::value, msg::expected),
+    "{value} is an unknown CPU architecture type, {expected} is the list of accepted CPU architecture values",
+    "Invalid architecture: {value}. Expected one of: {expected}")
 DECLARE_MESSAGE(InvalidArgument, (), "", "invalid argument")
 DECLARE_MESSAGE(
     InvalidArgumentRequiresAbsolutePath,
@@ -1954,6 +1968,11 @@ DECLARE_MESSAGE(InvalidOptionForRemove,
                 "'remove' is a command that should not be changed.",
                 "'remove' accepts either libraries or '--outdated'")
 DECLARE_MESSAGE(InvalidPortVersonName, (msg::path), "", "Found invalid port version file name: `{path}`.")
+DECLARE_MESSAGE(InvalidSha512,
+                (msg::sha),
+                "",
+                "invalid SHA-512 hash: {sha}\n"
+                "SHA-512 hash must be 128 characters long and contain only hexadecimal digits")
 DECLARE_MESSAGE(InvalidSharpInVersion, (), "", "invalid character '#' in version text")
 DECLARE_MESSAGE(InvalidSharpInVersionDidYouMean,
                 (msg::value),
@@ -3018,11 +3037,6 @@ DECLARE_MESSAGE(VersionCommandHeader,
                 (msg::version),
                 "",
                 "vcpkg package management program version {version}\n\nSee LICENSE.txt for license information.")
-DECLARE_MESSAGE(
-    VersionConflictXML,
-    (msg::path, msg::expected_version, msg::actual_version),
-    "",
-    "Expected {path} version: [{expected_version}], but was [{actual_version}]. Please re-run bootstrap-vcpkg.")
 DECLARE_MESSAGE(VersionConstraintNotInDatabase1,
                 (msg::package_name, msg::version),
                 "",

include/vcpkg/tools.h

@@ -62,7 +62,7 @@ namespace vcpkg
     std::unique_ptr<ToolCache> get_tool_cache(const Filesystem& fs,
                                               std::shared_ptr<const DownloadManager> downloader,
                                               Path downloads,
-                                              Path xml_config,
+                                              Path config_path,
                                               Path tools,
                                               RequireExactVersions abiToolVersionHandling);
 }

include/vcpkg/tools.test.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/fwd/expected.h>
+#include <vcpkg/base/fwd/json.h>
 #include <vcpkg/base/fwd/optional.h>
 #include <vcpkg/base/fwd/stringview.h>
 
@@ -29,7 +30,36 @@ namespace vcpkg
         Path exe_path(const Path& tools_base_path) const { return tools_base_path / tool_dir_subpath / exe_subpath; }
     };
 
-    Optional<ToolData> parse_tool_data_from_xml(StringView XML, StringView XML_PATH, StringView tool, StringView os);
-
     Optional<std::array<int, 3>> parse_tool_version_string(StringView string_version);
+
+    struct ToolDataEntry
+    {
+        std::string tool;
+        std::string os;
+        Optional<vcpkg::CPUArchitecture> arch;
+        std::string version;
+        std::string exeRelativePath;
+        std::string url;
+        std::string sha512;
+        std::string archiveName;
+    };
+
+    ExpectedL<std::vector<ToolDataEntry>> parse_tool_data(StringView contents, StringView origin);
+
+    const ToolDataEntry* get_raw_tool_data(const std::vector<ToolDataEntry>& tool_data_table,
+                                           StringView toolname,
+                                           const CPUArchitecture arch,
+                                           StringView os);
+
+    struct ToolDataFileDeserializer final : Json::IDeserializer<std::vector<ToolDataEntry>>
+    {
+        virtual LocalizedString type_name() const override;
+
+        virtual View<StringView> valid_fields() const override;
+
+        virtual Optional<std::vector<ToolDataEntry>> visit_object(Json::Reader& r,
+                                                                  const Json::Object& obj) const override;
+
+        static const ToolDataFileDeserializer instance;
+    };
 }

include/vcpkg/vcpkgcmdarguments.h

@@ -223,6 +223,7 @@ namespace vcpkg
         Optional<std::string> builtin_ports_root_dir;
         Optional<std::string> builtin_registry_versions_dir;
         Optional<std::string> registries_cache_dir;
+        Optional<std::string> tools_data_file;
 
         Optional<std::string> default_visual_studio_path;