wip

microsoft · Dec 13, 2024 · 4d8eb9b · 4d8eb9b
1 parent 6defa28
commit 4d8eb9b
Show file tree

Hide file tree

Showing 8 changed files with 459 additions and 422 deletions.
diff --git a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 b/azure-pipelines/end-to-end-tests-dir/fetch.ps1
@@ -6,42 +6,104 @@ if (-not $IsMacOS -and -not $IsLinux) {
     $Scripts = Join-Path $TestingRoot "scripts"
     mkdir $Scripts | Out-Null
 
-    $7zip_version = "24.08"
-    $ninja_version = "1.11.1"
+    $7zip_version = "19.00"
+    $ninja_version = "1.10.2"
+
+    @'
+[{
+    "name": "7zip",
+    "os": "windows",
+    "version": "19.00",
+    "executable": "Files\\7-Zip\\7z.exe",
+    "url": "https://www.7-zip.org/a/7z1900-x64.msi",
+    "sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8",
+    "archive": "7z1900-x64.msi"
+},
+{
+    "name": "ninja-testing",
+    "os": "windows",
+    "version": "1.10.2",
+    "executable": "ninja.exe",
+    "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+    "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+    "archive": "ninja-win-1.10.2.zip"
+},
+{
+    "name": "ninja",
+    "os": "windows",
+    "version": "1.10.2",
+    "executable": "ninja.exe",
+    "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+    "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+    "archive": "ninja-win-1.10.2.zip"
+},
+{
+    "name": "cmake",
+    "os": "windows",
+    "version": "3.22.2",
+    "executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe",
+    "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip",
+    "sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79",
+    "archive": "cmake-3.22.2-windows-i386.zip"
+},
+{
+    "name": "cmake",
+    "os": "osx",
+    "version": "3.22.2",
+    "executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake",
+    "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz",
+    "sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac",
+    "archive": "cmake-3.22.2-macos-universal.tar.gz"
+},
+{
+    "name": "cmake",
+    "os": "linux",
+    "version": "3.22.2",
+    "executable": "cmake-3.22.2-linux-x86_64/bin/cmake",
+    "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz",
+    "sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b",
+    "archive": "cmake-3.22.2linux-x86_64.tar.gz"
+},
+{
+    "name": "cmake",
+    "os": "freebsd",
+    "version": "3.20.4",
+    "executable": "/usr/local/bin/cmake",
+    "url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz",
+    "sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475",
+    "archive": "cmake-3.20.4.txz"
+}]
+'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkg-tools.json")
 
     $env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads'
-    $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/7zip-${7zip_version}-windows/7za.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
 
-    Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
+    Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe"
 
-    mkdir "$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows" | Out-Null
-    Move-Item -Path "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" -Destination "$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows/ninja.exe"
     $path = $env:PATH
-    Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES
 
-    $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows"
+    $env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
-    Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows" -ErrorAction SilentlyContinue
+    Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
     Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES
 
     $env:VCPKG_FORCE_SYSTEM_BINARIES = "1"
     $env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES
     $out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status"))

diff --git a/include/vcpkg/base/jsonreader.h b/include/vcpkg/base/jsonreader.h
@@ -361,4 +361,18 @@ namespace vcpkg::Json
         virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
         static const FeatureNameDeserializer instance;
     };
+
+    struct ArchitectureDeserializer final : Json::IDeserializer<std::string>
+    {
+        virtual LocalizedString type_name() const override;
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
+        static const ArchitectureDeserializer instance;
+    };
+
+    struct Sha512Deserializer final : Json::IDeserializer<std::string>
+    {
+        virtual LocalizedString type_name() const override;
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
+        static const Sha512Deserializer instance;
+    };
 }
diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h
@@ -62,6 +62,7 @@ namespace vcpkg
     std::unique_ptr<ToolCache> get_tool_cache(const Filesystem& fs,
                                               std::shared_ptr<const DownloadManager> downloader,
                                               Path downloads,
+                                              Path config_path,
                                               Path tools,
                                               RequireExactVersions abiToolVersionHandling);
 }
diff --git a/include/vcpkg/tools.test.h b/include/vcpkg/tools.test.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/fwd/expected.h>
+#include <vcpkg/base/fwd/json.h>
 #include <vcpkg/base/fwd/optional.h>
 #include <vcpkg/base/fwd/stringview.h>
 
@@ -30,4 +31,35 @@ namespace vcpkg
     };
 
     Optional<std::array<int, 3>> parse_tool_version_string(StringView string_version);
+
+    struct ArchToolData
+    {
+        std::string tool;
+        std::string os;
+        Optional<vcpkg::CPUArchitecture> arch;
+        std::string version;
+        std::string exeRelativePath;
+        std::string url;
+        std::string sha512;
+        std::string archiveName;
+    };
+
+    const ArchToolData* get_raw_tool_data(const std::vector<ArchToolData>& tool_data_table,
+                                          StringView toolname,
+                                          const CPUArchitecture arch,
+                                          StringView os);
+
+    struct ToolDataDeserializer final : Json::IDeserializer<ArchToolData>
+    {
+        virtual LocalizedString type_name() const override;
+
+        virtual Optional<ArchToolData> visit_object(Json::Reader& r, const Json::Object& obj) const override;
+        static const ToolDataDeserializer instance;
+    };
+
+    struct ToolDataArrayDeserializer final : Json::ArrayDeserializer<ToolDataDeserializer>
+    {
+        virtual LocalizedString type_name() const override;
+        static const ToolDataArrayDeserializer instance;
+    };
 }
diff --git a/src/vcpkg-test/tools.cpp b/src/vcpkg-test/tools.cpp
@@ -1,10 +1,31 @@
 #include <vcpkg-test/util.h>
 
+#include <vcpkg/base/jsonreader.h>
+
 #include <vcpkg/tools.h>
 #include <vcpkg/tools.test.h>
 
 using namespace vcpkg;
 
+namespace
+{
+    std::vector<ArchToolData> parse_tool_data_json(StringView json_contents, StringView origin)
+    {
+        auto as_json = Json::parse(json_contents, origin);
+        REQUIRE(as_json.has_value());
+
+        auto as_value = std::move(as_json).value(VCPKG_LINE_INFO).value;
+        REQUIRE(as_value.is_array());
+
+        Json::Reader r(origin);
+        ToolDataArrayDeserializer tool_data_deserializer;
+        auto maybe_tool_data = r.visit(as_value, tool_data_deserializer);
+        REQUIRE(maybe_tool_data.has_value());
+
+        return std::move(*maybe_tool_data.get());
+    }
+}
+
 TEST_CASE ("parse_tool_version_string", "[tools]")
 {
     auto result = parse_tool_version_string("1.2.3");
@@ -65,3 +86,126 @@ TEST_CASE ("extract_prefixed_nonquote", "[tools]")
     CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to "
                                   "determine the version:\nmalformed output");
 }
+
+TEST_CASE ("parse_tool_data_from_json", "[tools]")
+{
+    const StringView tool_doc = R"([
+{
+    "name": "git",
+    "os": "linux",
+    "version": "2.7.4",
+    "executable": "git"
+},
+{
+    "name": "git",
+    "os": "linux",
+    "arch": "arm64",
+    "version": "2.7.4",
+    "executable": "git-arm64"
+},
+{
+    "name": "nuget",
+    "os": "osx",
+    "version": "5.11.0",
+    "executable": "nuget.exe",
+    "url": "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe",
+    "sha512": "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda83b64c03ce14c8f35ca9957a17a8c02b8c4b7"
+},
+{
+    "name": "node",
+    "os": "windows",
+    "version": "16.12.0",
+    "executable": "node-v16.12.0-win-x64\\node.exe",
+    "url": "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z",
+    "sha512": "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023fb7068ed1f8a7678e446260c3db3537fa888",
+    "archive": "node-v16.12.0-win-x64.7z"
+}
+])";
+
+    auto data = parse_tool_data_json(tool_doc, "vcpkgTools.json");
+    REQUIRE(data.size() == 4);
+
+    auto git_linux = data[0];
+    CHECK(git_linux.tool == "git");
+    CHECK(git_linux.os == "linux");
+    CHECK_FALSE(git_linux.arch.has_value());
+    CHECK(git_linux.version == "2.7.4");
+    CHECK(git_linux.exeRelativePath == "git");
+    CHECK(git_linux.url == "");
+    CHECK(git_linux.sha512 == "");
+
+    auto git_arm64 = data[1];
+    CHECK(git_arm64.tool == "git");
+    CHECK(git_arm64.os == "linux");
+    CHECK(git_arm64.arch.has_value());
+    CHECK(*git_arm64.arch.get() == CPUArchitecture::ARM64);
+    CHECK(git_arm64.version == "2.7.4");
+    CHECK(git_arm64.exeRelativePath == "git-arm64");
+    CHECK(git_arm64.url == "");
+    CHECK(git_arm64.sha512 == "");
+
+    auto nuget_osx = data[2];
+    CHECK(nuget_osx.tool == "nuget");
+    CHECK(nuget_osx.os == "osx");
+    CHECK_FALSE(nuget_osx.arch.has_value());
+    CHECK(nuget_osx.version == "5.11.0");
+    CHECK(nuget_osx.exeRelativePath == "nuget.exe");
+    CHECK(nuget_osx.url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe");
+    CHECK(nuget_osx.sha512 == "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9eb"
+                              "fda83b64c03ce14c8f35ca9957a17a8c02b8c4b7");
+
+    auto node_windows = data[3];
+    CHECK(node_windows.tool == "node");
+    CHECK(node_windows.os == "windows");
+    CHECK_FALSE(node_windows.arch.has_value());
+    CHECK(node_windows.version == "16.12.0");
+    CHECK(node_windows.exeRelativePath == "node-v16.12.0-win-x64\\node.exe");
+    CHECK(node_windows.url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z");
+    CHECK(node_windows.sha512 ==
+          "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023"
+          "fb7068ed1f8a7678e446260c3db3537fa888");
+    CHECK(node_windows.archiveName == "node-v16.12.0-win-x64.7z");
+
+    auto* tooL_git_linux = get_raw_tool_data(data, "git", CPUArchitecture::X64, "linux");
+    REQUIRE(tooL_git_linux != nullptr);
+    CHECK(tooL_git_linux->tool == "git");
+    CHECK(tooL_git_linux->os == "linux");
+    CHECK_FALSE(tooL_git_linux->arch.has_value());
+    CHECK(tooL_git_linux->version == "2.7.4");
+    CHECK(tooL_git_linux->exeRelativePath == "git");
+    CHECK(tooL_git_linux->url == "");
+    CHECK(tooL_git_linux->sha512 == "");
+
+    auto* tooL_git_arm64 = get_raw_tool_data(data, "git", CPUArchitecture::ARM64, "linux");
+    REQUIRE(tooL_git_arm64 != nullptr);
+    CHECK(tooL_git_arm64->tool == "git");
+    CHECK(tooL_git_arm64->os == "linux");
+    CHECK(tooL_git_arm64->arch.has_value());
+    CHECK(*tooL_git_arm64->arch.get() == CPUArchitecture::ARM64);
+    CHECK(tooL_git_arm64->version == "2.7.4");
+    CHECK(tooL_git_arm64->exeRelativePath == "git-arm64");
+    CHECK(tooL_git_arm64->url == "");
+    CHECK(tooL_git_arm64->sha512 == "");
+
+    auto* tooL_nuget_osx = get_raw_tool_data(data, "nuget", CPUArchitecture::X64, "osx");
+    REQUIRE(tooL_nuget_osx != nullptr);
+    CHECK(tooL_nuget_osx->tool == "nuget");
+    CHECK(tooL_nuget_osx->os == "osx");
+    CHECK_FALSE(tooL_nuget_osx->arch.has_value());
+    CHECK(tooL_nuget_osx->version == "5.11.0");
+    CHECK(tooL_nuget_osx->exeRelativePath == "nuget.exe");
+    CHECK(tooL_nuget_osx->url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe");
+
+    auto* tooL_node_windows = get_raw_tool_data(data, "node", CPUArchitecture::X64, "windows");
+    REQUIRE(tooL_node_windows != nullptr);
+    CHECK(tooL_node_windows->tool == "node");
+    CHECK(tooL_node_windows->os == "windows");
+    CHECK_FALSE(tooL_node_windows->arch.has_value());
+    CHECK(tooL_node_windows->version == "16.12.0");
+    CHECK(tooL_node_windows->exeRelativePath == "node-v16.12.0-win-x64\\node.exe");
+    CHECK(tooL_node_windows->url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z");
+    CHECK(tooL_node_windows->sha512 ==
+          "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023"
+          "fb7068ed1f8a7678e446260c3db3537fa888");
+    CHECK(tooL_node_windows->archiveName == "node-v16.12.0-win-x64.7z");
+}