diff --git a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 index 1ed49fc174..cd83a52c54 100644 --- a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/fetch.ps1 @@ -6,42 +6,104 @@ if (-not $IsMacOS -and -not $IsLinux) { $Scripts = Join-Path $TestingRoot "scripts" mkdir $Scripts | Out-Null - $7zip_version = "24.08" - $ninja_version = "1.11.1" + $7zip_version = "19.00" + $ninja_version = "1.10.2" + + @' +[{ + "name": "7zip", + "os": "windows", + "version": "19.00", + "executable": "Files\\7-Zip\\7z.exe", + "url": "https://www.7-zip.org/a/7z1900-x64.msi", + "sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8", + "archive": "7z1900-x64.msi" +}, +{ + "name": "ninja-testing", + "os": "windows", + "version": "1.10.2", + "executable": "ninja.exe", + "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip", + "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3", + "archive": "ninja-win-1.10.2.zip" +}, +{ + "name": "ninja", + "os": "windows", + "version": "1.10.2", + "executable": "ninja.exe", + "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip", + "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3", + "archive": "ninja-win-1.10.2.zip" +}, +{ + "name": "cmake", + "os": "windows", + "version": "3.22.2", + "executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip", + "sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79", + "archive": "cmake-3.22.2-windows-i386.zip" +}, +{ + "name": "cmake", + "os": "osx", + "version": "3.22.2", + "executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz", + "sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac", + "archive": "cmake-3.22.2-macos-universal.tar.gz" +}, +{ + "name": "cmake", + "os": "linux", + "version": "3.22.2", + "executable": "cmake-3.22.2-linux-x86_64/bin/cmake", + "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz", + "sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b", + "archive": "cmake-3.22.2linux-x86_64.tar.gz" +}, +{ + "name": "cmake", + "os": "freebsd", + "version": "3.20.4", + "executable": "/usr/local/bin/cmake", + "url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz", + "sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475", + "archive": "cmake-3.20.4.txz" +}] +'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkg-tools.json") $env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads' - $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/7zip-${7zip_version}-windows/7za.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe" - Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) + Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe" - mkdir "$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows" | Out-Null - Move-Item -Path "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" -Destination "$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows/ninja.exe" $path = $env:PATH - Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES - $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-${ninja_version}-windows" + $env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileNotExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" + Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" + Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" - Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows" -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES $env:VCPKG_FORCE_SYSTEM_BINARIES = "1" $env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path" Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot")) Throw-IfFailed - Require-FileNotExists "$TestingRoot/down loads/tools/ninja-${ninja_version}-windows/ninja.exe" + Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe" Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES $out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status")) diff --git a/include/vcpkg/base/jsonreader.h b/include/vcpkg/base/jsonreader.h index e4c3522ed7..e1bc62172f 100644 --- a/include/vcpkg/base/jsonreader.h +++ b/include/vcpkg/base/jsonreader.h @@ -361,4 +361,18 @@ namespace vcpkg::Json virtual Optional visit_string(Json::Reader&, StringView sv) const override; static const FeatureNameDeserializer instance; }; + + struct ArchitectureDeserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + virtual Optional visit_string(Json::Reader&, StringView sv) const override; + static const ArchitectureDeserializer instance; + }; + + struct Sha512Deserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + virtual Optional visit_string(Json::Reader&, StringView sv) const override; + static const Sha512Deserializer instance; + }; } diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h index 7f3819a403..6542f324e3 100644 --- a/include/vcpkg/tools.h +++ b/include/vcpkg/tools.h @@ -62,6 +62,7 @@ namespace vcpkg std::unique_ptr get_tool_cache(const Filesystem& fs, std::shared_ptr downloader, Path downloads, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling); } diff --git a/include/vcpkg/tools.test.h b/include/vcpkg/tools.test.h index 62a47eeca1..2a2003dbc4 100644 --- a/include/vcpkg/tools.test.h +++ b/include/vcpkg/tools.test.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -30,4 +31,35 @@ namespace vcpkg }; Optional> parse_tool_version_string(StringView string_version); + + struct ArchToolData + { + std::string tool; + std::string os; + Optional arch; + std::string version; + std::string exeRelativePath; + std::string url; + std::string sha512; + std::string archiveName; + }; + + const ArchToolData* get_raw_tool_data(const std::vector& tool_data_table, + StringView toolname, + const CPUArchitecture arch, + StringView os); + + struct ToolDataDeserializer final : Json::IDeserializer + { + virtual LocalizedString type_name() const override; + + virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) const override; + static const ToolDataDeserializer instance; + }; + + struct ToolDataArrayDeserializer final : Json::ArrayDeserializer + { + virtual LocalizedString type_name() const override; + static const ToolDataArrayDeserializer instance; + }; } diff --git a/src/vcpkg-test/tools.cpp b/src/vcpkg-test/tools.cpp index 055f0fcff3..ddb4a712cf 100644 --- a/src/vcpkg-test/tools.cpp +++ b/src/vcpkg-test/tools.cpp @@ -1,10 +1,31 @@ #include +#include + #include #include using namespace vcpkg; +namespace +{ + std::vector parse_tool_data_json(StringView json_contents, StringView origin) + { + auto as_json = Json::parse(json_contents, origin); + REQUIRE(as_json.has_value()); + + auto as_value = std::move(as_json).value(VCPKG_LINE_INFO).value; + REQUIRE(as_value.is_array()); + + Json::Reader r(origin); + ToolDataArrayDeserializer tool_data_deserializer; + auto maybe_tool_data = r.visit(as_value, tool_data_deserializer); + REQUIRE(maybe_tool_data.has_value()); + + return std::move(*maybe_tool_data.get()); + } +} + TEST_CASE ("parse_tool_version_string", "[tools]") { auto result = parse_tool_version_string("1.2.3"); @@ -65,3 +86,126 @@ TEST_CASE ("extract_prefixed_nonquote", "[tools]") CHECK(error_result.error() == "error: fooutil (fooutil.exe) produced unexpected output when attempting to " "determine the version:\nmalformed output"); } + +TEST_CASE ("parse_tool_data_from_json", "[tools]") +{ + const StringView tool_doc = R"([ +{ + "name": "git", + "os": "linux", + "version": "2.7.4", + "executable": "git" +}, +{ + "name": "git", + "os": "linux", + "arch": "arm64", + "version": "2.7.4", + "executable": "git-arm64" +}, +{ + "name": "nuget", + "os": "osx", + "version": "5.11.0", + "executable": "nuget.exe", + "url": "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe", + "sha512": "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9ebfda83b64c03ce14c8f35ca9957a17a8c02b8c4b7" +}, +{ + "name": "node", + "os": "windows", + "version": "16.12.0", + "executable": "node-v16.12.0-win-x64\\node.exe", + "url": "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z", + "sha512": "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023fb7068ed1f8a7678e446260c3db3537fa888", + "archive": "node-v16.12.0-win-x64.7z" +} +])"; + + auto data = parse_tool_data_json(tool_doc, "vcpkgTools.json"); + REQUIRE(data.size() == 4); + + auto git_linux = data[0]; + CHECK(git_linux.tool == "git"); + CHECK(git_linux.os == "linux"); + CHECK_FALSE(git_linux.arch.has_value()); + CHECK(git_linux.version == "2.7.4"); + CHECK(git_linux.exeRelativePath == "git"); + CHECK(git_linux.url == ""); + CHECK(git_linux.sha512 == ""); + + auto git_arm64 = data[1]; + CHECK(git_arm64.tool == "git"); + CHECK(git_arm64.os == "linux"); + CHECK(git_arm64.arch.has_value()); + CHECK(*git_arm64.arch.get() == CPUArchitecture::ARM64); + CHECK(git_arm64.version == "2.7.4"); + CHECK(git_arm64.exeRelativePath == "git-arm64"); + CHECK(git_arm64.url == ""); + CHECK(git_arm64.sha512 == ""); + + auto nuget_osx = data[2]; + CHECK(nuget_osx.tool == "nuget"); + CHECK(nuget_osx.os == "osx"); + CHECK_FALSE(nuget_osx.arch.has_value()); + CHECK(nuget_osx.version == "5.11.0"); + CHECK(nuget_osx.exeRelativePath == "nuget.exe"); + CHECK(nuget_osx.url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); + CHECK(nuget_osx.sha512 == "06a337c9404dec392709834ef2cdbdce611e104b510ef40201849595d46d242151749aef65bc2d7ce5ade9eb" + "fda83b64c03ce14c8f35ca9957a17a8c02b8c4b7"); + + auto node_windows = data[3]; + CHECK(node_windows.tool == "node"); + CHECK(node_windows.os == "windows"); + CHECK_FALSE(node_windows.arch.has_value()); + CHECK(node_windows.version == "16.12.0"); + CHECK(node_windows.exeRelativePath == "node-v16.12.0-win-x64\\node.exe"); + CHECK(node_windows.url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); + CHECK(node_windows.sha512 == + "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" + "fb7068ed1f8a7678e446260c3db3537fa888"); + CHECK(node_windows.archiveName == "node-v16.12.0-win-x64.7z"); + + auto* tooL_git_linux = get_raw_tool_data(data, "git", CPUArchitecture::X64, "linux"); + REQUIRE(tooL_git_linux != nullptr); + CHECK(tooL_git_linux->tool == "git"); + CHECK(tooL_git_linux->os == "linux"); + CHECK_FALSE(tooL_git_linux->arch.has_value()); + CHECK(tooL_git_linux->version == "2.7.4"); + CHECK(tooL_git_linux->exeRelativePath == "git"); + CHECK(tooL_git_linux->url == ""); + CHECK(tooL_git_linux->sha512 == ""); + + auto* tooL_git_arm64 = get_raw_tool_data(data, "git", CPUArchitecture::ARM64, "linux"); + REQUIRE(tooL_git_arm64 != nullptr); + CHECK(tooL_git_arm64->tool == "git"); + CHECK(tooL_git_arm64->os == "linux"); + CHECK(tooL_git_arm64->arch.has_value()); + CHECK(*tooL_git_arm64->arch.get() == CPUArchitecture::ARM64); + CHECK(tooL_git_arm64->version == "2.7.4"); + CHECK(tooL_git_arm64->exeRelativePath == "git-arm64"); + CHECK(tooL_git_arm64->url == ""); + CHECK(tooL_git_arm64->sha512 == ""); + + auto* tooL_nuget_osx = get_raw_tool_data(data, "nuget", CPUArchitecture::X64, "osx"); + REQUIRE(tooL_nuget_osx != nullptr); + CHECK(tooL_nuget_osx->tool == "nuget"); + CHECK(tooL_nuget_osx->os == "osx"); + CHECK_FALSE(tooL_nuget_osx->arch.has_value()); + CHECK(tooL_nuget_osx->version == "5.11.0"); + CHECK(tooL_nuget_osx->exeRelativePath == "nuget.exe"); + CHECK(tooL_nuget_osx->url == "https://dist.nuget.org/win-x86-commandline/v5.11.0/nuget.exe"); + + auto* tooL_node_windows = get_raw_tool_data(data, "node", CPUArchitecture::X64, "windows"); + REQUIRE(tooL_node_windows != nullptr); + CHECK(tooL_node_windows->tool == "node"); + CHECK(tooL_node_windows->os == "windows"); + CHECK_FALSE(tooL_node_windows->arch.has_value()); + CHECK(tooL_node_windows->version == "16.12.0"); + CHECK(tooL_node_windows->exeRelativePath == "node-v16.12.0-win-x64\\node.exe"); + CHECK(tooL_node_windows->url == "https://nodejs.org/dist/v16.12.0/node-v16.12.0-win-x64.7z"); + CHECK(tooL_node_windows->sha512 == + "0bb793fce8140bd59c17f3ac9661b062eac0f611d704117774f5cb2453d717da94b1e8b17d021d47baff598dc023" + "fb7068ed1f8a7678e446260c3db3537fa888"); + CHECK(tooL_node_windows->archiveName == "node-v16.12.0-win-x64.7z"); +} diff --git a/src/vcpkg/base/json.cpp b/src/vcpkg/base/json.cpp index e65c51e65a..38b94294ff 100644 --- a/src/vcpkg/base/json.cpp +++ b/src/vcpkg/base/json.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -1569,4 +1570,58 @@ namespace vcpkg::Json } const FeatureNameDeserializer FeatureNameDeserializer::instance; + + LocalizedString ArchitectureDeserializer::type_name() const + { + // TODO: Add type name + return msg::format(msgAFeatureName); + } + + Optional ArchitectureDeserializer::visit_string(Json::Reader& r, StringView sv) const + { + static const std::vector known_architectures{ + "x86", + "x64", + "arm", + "arm64", + "arm64ec", + "s390x", + "ppc64le", + "riscv32", + "riscv64", + "loongarch32", + "loongarch64", + "mips64", + }; + + if (Util::contains(known_architectures, sv)) + { + return sv.to_string(); + } + + r.add_generic_error(type_name(), msg::format(msgInvalidArchitecture, msg::value = sv)); + return nullopt; + } + + const ArchitectureDeserializer ArchitectureDeserializer::instance; + + LocalizedString Sha512Deserializer::type_name() const + { + // TODO: Add type name + return msg::format(msgAFeatureName); + } + + Optional Sha512Deserializer::visit_string(Json::Reader&, StringView sv) const + { + if (sv.size() == 128 && std::all_of(sv.begin(), sv.end(), ParserBase::is_hex_digit)) + { + return sv.to_string(); + } + + // TODO: Add error + // r.add_generic_error(type_name(), msg::format(msgInvalidSha512Length, msg::value = sv)); + return nullopt; + } + + const Sha512Deserializer Sha512Deserializer::instance; } diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 6e4015f8da..35235a6a9e 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -62,441 +64,96 @@ namespace vcpkg return std::array{*d1.get(), *d2.get(), *d3.get()}; } - struct ArchToolData + ExpectedL> parse_tool_data(const Filesystem& fs, Path path) { - StringLiteral tool; - StringLiteral os; - Optional arch; - StringLiteral version; - StringLiteral exeRelativePath; - StringLiteral url; - StringLiteral sha512; - StringLiteral archiveName; - }; - - static constexpr ArchToolData tool_data_table[] = { - { - "python3", - "windows", - nullopt, - "3.11.8", - "python.exe", - "https://www.python.org/ftp/python/3.11.8/python-3.11.8-embed-win32.zip", - "c88ef02f0860000dbc59361cfe051e3e8dc7d208ed39bb5bc20a3e8b8711b578926e281a11941787ea61b2ef05b945ab313332" - "2dcb85b916f79ac4ada57f6309", - "python-3.11.8-embed-win32.zip", - }, - { - "python3_with_venv", - "windows", - nullopt, - "3.11.8", - "tools/python.exe", - "https://www.nuget.org/api/v2/package/python/3.11.8", - "8c1abd622fb2795fb87ab5208764fe13f7f4d69f2283b4237ea47b7c14b0989b5b4169f1ac1d3b1f417c6c2cc3d85ce151370d" - "ae8efc807c93e57cd67fa3c8cf", - "python-3.11.8.nupkg.zip", - }, - { - "cmake", - "windows", - nullopt, - "3.29.2", - "cmake-3.29.2-windows-i386/bin/cmake.exe", - "https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-windows-i386.zip", - "562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42b69e4aeb31a4879e1bcf6f7c2d41f2a" - "ce12180ea83ba7af48879d40ab", - "cmake-3.29.2-windows-i386.zip", - }, - { - "cmake", - "osx", - nullopt, - "3.29.2", - "cmake-3.29.2-macos-universal/CMake.app/Contents/bin/cmake", - "https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-macos-universal.tar.gz", - "26aab0163965f3d560dfd6b1f72c5e77192338237ebe286099fd62f243f1bbd4857b9193118386b801c00dc5cfbc5bc8af8481" - "4692fcfadcf56c7d2faab52533", - "cmake-3.29.2-macos-universal.tar.gz", - }, - { - "cmake", - "linux", - CPUArchitecture::ARM64, - "3.29.2", - "cmake-3.29.2-linux-aarch64/bin/cmake", - "https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-aarch64.tar.gz", - "206b00604caf72f3dc644c7a5ac6a335814520fbf1512b9f2d4a5e8c26025d286bd106d2925ffbc874c754c518cbdb07f16919" - "16c39fcfd1202a47f592c8e3b0", - "cmake-3.29.2-linux-aarch64.tar.gz", - }, - { - "cmake", - "linux", - nullopt, - "3.29.2", - "cmake-3.29.2-linux-x86_64/bin/cmake", - "https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz", - "d88082d582f1774a3d89efbf3f31a03b08a663c359d603d87ee7c4afd862d4164d2b0b9f0b32cece8efee3acfa86b44214cd4f" - "7606b99b8334183a05d5f06edc", - "cmake-3.29.2-linux-x86_64.tar.gz", - }, - { - "git", - "windows", - nullopt, - "2.7.4", - "mingw64/bin/git.exe", - "https://github.com/git-for-windows/git/releases/download/v2.43.0.windows.1/" - "PortableGit-2.43.0-64-bit.7z.exe", - "02ec40f55a6de18f305530e9415b6ce0a597359ebb9a58cf727ac84eceb0003b0f437941b76872b6568157a333c8e6e8d865a3" - "6faf874fd5f04774deb6a9387a", - "PortableGit-2.43.0-32-bit.7z.exe", - }, - { - "git", - "linux", - nullopt, - "2.7.4", - "", - "", - "", - "", - }, - { - "git", - "osx", - nullopt, - "2.7.4", - "", - "", - "", - "", - }, - { - "git", - "freebsd", - nullopt, - "2.7.4", - "", - "", - "", - "", - }, - { - "gsutil", - "windows", - nullopt, - "4.65", - "google-cloud-sdk/bin/gsutil.cmd", - "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/" - "google-cloud-sdk-347.0.0-windows-x86_64-bundled-python.zip", - "e2792e17b132aad77f7c0b9fd26faf415e9437923d9227a9e6d253554e6843d29a6ddad0a7fb5e9aea4a130fd4c521e6ece884" - "4fd4a4f9e8d580348775425389", - "google-cloud-sdk-347.0.0-windows-x86_64-bundled-python.zip", - }, - { - "gsutil", - "osx", - nullopt, - "4.65", - "gsutil/gsutil", - "https://storage.googleapis.com/pub/gsutil_4.65.tar.gz", - "2c5c9dea48147f97180a491bbb9e24e8cbcd4f3452620e2f80338b781e4dfc90bb754e3bbfa05e1b990e44bff52d990d8c2dd5" - "1bc83d112339d8e6096a2f21c8", - "gsutil_4.65.tar.gz", - }, - { - "gsutil", - "linux", - nullopt, - "4.65", - "gsutil/gsutil", - "https://storage.googleapis.com/pub/gsutil_4.65.tar.gz", - "2c5c9dea48147f97180a491bbb9e24e8cbcd4f3452620e2f80338b781e4dfc90bb754e3bbfa05e1b990e44bff52d990d8c2dd5" - "1bc83d112339d8e6096a2f21c8", - "gsutil_4.65.tar.gz", - }, - { - "vswhere", - "windows", - nullopt, - "3.1.7", - "vswhere.exe", - "https://github.com/microsoft/vswhere/releases/download/3.1.7/vswhere.exe", - "40c534eb27f079c15c9782f53f82c12dabfede4d3d85f0edf8a855c2b0d5e12921a96506b37c210beab3c33220f8ff098447ad" - "054e82d8c2603964975fc12076", - "", - }, - { - "nuget", - "windows", - nullopt, - "6.10.0", - "nuget.exe", - "https://dist.nuget.org/win-x86-commandline/v6.10.0/nuget.exe", - "71d7307bb89de2df3811419c561efa00618a4c68e6ce481b0bdfc94c7c6c6d126a54eb26a0015686fabf99f109744ca41fead9" - "9e97139cdc86dde16a5ec3e7cf", - "", - }, - { - "nuget", - "linux", - nullopt, - "6.10.0", - "nuget.exe", - "https://dist.nuget.org/win-x86-commandline/v6.10.0/nuget.exe", - "71d7307bb89de2df3811419c561efa00618a4c68e6ce481b0bdfc94c7c6c6d126a54eb26a0015686fabf99f109744ca41fead9" - "9e97139cdc86dde16a5ec3e7cf", - "", - }, + auto contents = fs.read_contents(path, IgnoreErrors{}); + if (contents.empty()) { - "nuget", - "osx", - nullopt, - "6.10.0", - "nuget.exe", - "https://dist.nuget.org/win-x86-commandline/v6.10.0/nuget.exe", - "71d7307bb89de2df3811419c561efa00618a4c68e6ce481b0bdfc94c7c6c6d126a54eb26a0015686fabf99f109744ca41fead9" - "9e97139cdc86dde16a5ec3e7cf", - "", - }, - { - "coscli", - "windows", - nullopt, - "0.11.0", - "coscli-windows.exe", - "https://github.com/tencentyun/coscli/releases/download/v0.11.0-beta/coscli-windows.exe", - "38a521ec80cdb6944124f4233d7e71eed8cc9f9be2c0c736269915d21c3718ea8131e4516bb6eeada6df331f5fa8f47a299907" - "e50ee9edbe0114444520974d06", - "", - }, - { - "coscli", - "linux", - nullopt, - "0.11.0", - "coscli-linux", - "https://github.com/tencentyun/coscli/releases/download/v0.11.0-beta/coscli-linux", - "9c930a1d308e9581a0e2fdfe3751ea7fe13d6068df90ca6465740ec3eda034202ef71ec54c99e90015ff81aa68aa1489567db5" - "e411e222eb7258704bdac7e924", - "", - }, - { - "coscli", - "osx", - nullopt, - "0.11.0", - "coscli-mac", - "https://github.com/tencentyun/coscli/releases/download/v0.11.0-beta/coscli-mac", - "9556335bfc8bc14bace6dfced45fa77fb07c80f08aa975e047a54efda1d19852aae0ea68a5bc7f04fbd88e3edce5a73512a612" - "16b1c5ff4cade224de4a9ab8db", - "", - }, - { - "installerbase", - "windows", - nullopt, - "4.4.0", - "QtInstallerFramework-win-x86/bin/installerbase.exe", - "https://download.qt.io/official_releases/qt-installer-framework/4.4.0/" - "installer-framework-opensource-src-4.4.0.zip", - "fc713f54bfe2781cb232cd0ae8eddb96833ec178d53a55ec0b01886aa048b13441eb49a1f33282e8eab7259cfe512c890d50b8" - "e632d3dbf501a0bf1fd83de947", - "installer-framework-opensource-src-4.4.0.zip", - }, - { - "7zip_msi", - "windows", - nullopt, - "24.08", - "Files/7-Zip/7z.exe", - "https://github.com/ip7z/7zip/releases/download/24.08/7z2408-x64.msi", - "3259bf5e251382333c9d18a3fc01d83491fb41bc4ac4ddb25a02918494594c1074482b6608189a8a89e343d78e34d57420cdef" - "f1d7ace5acfdcaacc8776f1be8", - "7z2408-x64.msi", - }, - { - "7zip", - "windows", - nullopt, - "24.08", - "7za.exe", - "https://github.com/ip7z/7zip/releases/download/24.08/7z2408-extra.7z", - "35f55236fccfb576ca014e29d0c35f4a213e53f06683bd2e82f869ed02506e230c8dd623c01d0207244d6a997031f737903456" - "b7ad4a44db1717f0a17a78602e", - "7z2408-extra.7z", - }, - { - "7zr", - "windows", - nullopt, - "24.08", - "7zr.exe", - "https://github.com/ip7z/7zip/releases/download/24.08/7zr.exe", - "424196f2acf5b89807f4038683acc50e7604223fc630245af6bab0e0df923f8b1c49cb09ac709086568c214c3f53dcb7d6c32e" - "8a54af222a3ff78cfab9c51670", - "", - }, - { - "aria2", - "windows", - nullopt, - "1.37.0", - "aria2-1.37.0-win-64bit-build1/aria2c.exe", - "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0-win-64bit-build1.zip", - "6d78405da9cf5639dbe8174787002161b8124d73880fb57cc8c0a3a63982f84e46df4e626990c58f23452965ad925f0d37cb91" - "47e99b25c3d7ca0ea49602f34d", - "aria2-1.37.0-win-64bit-build1.zip", - }, - { - "aria2", - "osx", - nullopt, - "1.35.0", - "aria2-1.35.0/bin/aria2c", - "https://github.com/aria2/aria2/releases/download/release-1.35.0/aria2-1.35.0-osx-darwin.tar.bz2", - "3bb32b7d55347d1af37c6f4ebf0e20b38ce51c37a1baf92f7ad1762000539a03413dd679a6d902fdb1805fa71917300c9692ac" - "eee012eb06ecdff10491137aec", - "aria2-1.35.0-osx-darwin.tar.bz2", - }, - { - "ninja", - "windows", - nullopt, - "1.11.1", - "ninja.exe", - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip", - "a700e794c32eb67b9f87040db7f1ba3a8e891636696fc54d416b01661c2421ff46fa517c97fd904adacdf8e621df3e68ea3801" - "05b909ae8b6651a78ae7eb3199", - "ninja-win-1.11.1.zip", - }, - { - "ninja", - "linux", - nullopt, - "1.11.1", - "ninja", - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-linux.zip", - "6403dac9196baffcff614fa73ea530752997c8db6bbfbaa0446b4b09d7327e2aa6e8615d1283c961d3bf0df497e85ba8660414" - "9f1505ee75f89d600245a45dde", - "ninja-linux-1.11.1.zip", - }, - { - "ninja", - "osx", - nullopt, - "1.11.1", - "ninja", - "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-mac.zip", - "dad33b0918c60bbf5107951a936175b1610b4894a408f4ba4b47a2f5b328fc982a52a2aed6a0cb75028ee4765af5083bea6661" - "1c37516826eb0c851366bb4427", - "ninja-mac-1.11.1.zip", - }, + // TODO: Add an error message + return msg::format_error(LocalizedString::from_raw("")); + } + + auto as_json = Json::parse(contents, path); + if (!as_json) { - "powershell-core", - "windows", - nullopt, - "7.2.23", - "pwsh.exe", - "https://github.com/PowerShell/PowerShell/releases/download/v7.2.23/PowerShell-7.2.23-win-x64.zip", - "b374a878df02980d54e17ad7cfc9021e331748c3770f586be61356c257494d1b33899c1167d09a35c210bc084474aefdff972f" - "672d16afb43be0562b3589285a", - "PowerShell-7.2.23-win-x64.zip", - }, + // TODO: Add an error message + return msg::format_error(LocalizedString::from_raw("")); + } + + auto as_value = std::move(as_json).value(VCPKG_LINE_INFO).value; + if (!as_value.is_array()) { - "node", - "windows", - nullopt, - "16.15.1", - "node-v16.15.1-win-x64/node.exe", - "https://nodejs.org/dist/v16.15.1/node-v16.15.1-win-x64.7z", - "7ec4bfe2ea6034e1461e306b6372d62c0c5d1060c453ba76a73a5cec38ac26b5952a744caa9071455329caa58eb0a96d26c688" - "54c8915c17610ff27b0cf2c1cf", - "node-v16.15.1-win-x64.7z", - }, + // TODO: Add an error message + return msg::format_error(LocalizedString::from_raw("")); + } + + Json::Reader r(path); + auto maybe_tool_data = r.visit(as_value, ToolDataArrayDeserializer::instance); + + if (!r.errors().empty()) { - "node", - "linux", - nullopt, - "16.15.1", - "node-v16.15.1-linux-x64/bin/node", - "https://nodejs.org/dist/v16.15.1/node-v16.15.1-linux-x64.tar.gz", - "5ad3b4b9caeaa8d31503efa99f5a593118a267dec9d4181d019732126ba248ce9a901207115b3f6b899eb5b3f0373c7f77ea95" - "cc92ac625cddf437ee9b8b8919", - "node-v16.15.1-linux-x64.tar.gz", - }, + // TODO: Extract error messages + for (auto&& e : r.errors()) + { + msg::println_error(e); + } + return msg::format_error(LocalizedString::from_raw("error parsing tools json file")); + } + + if (auto tool_data = maybe_tool_data.get()) { - "node", - "osx", - nullopt, - "16.15.1", - "node-v16.15.1-darwin-x64/bin/node", - "https://nodejs.org/dist/v16.15.1/node-v16.15.1-darwin-x64.tar.gz", - "90d0612bbe5467b6cf385c91a68b8daad0057e3e0ccacea44567f5b95b14f7481cb79784185ab1463b4bd990e092ff0f910957" - "6d1a1934b84e1c816582929611", - "node-v16.15.1-darwin-x64.tar.gz", - }, - }; + return *tool_data; + } + + // TODO: Add an error message + return msg::format_error(LocalizedString::from_raw("")); + } - static const ArchToolData* get_raw_tool_data(StringView toolname, CPUArchitecture arch, StringView os) + const ArchToolData* get_raw_tool_data(const std::vector& tool_data_table, + StringView toolname, + const CPUArchitecture arch, + StringView os) { const ArchToolData* default_tool = nullptr; - for (auto&& d : tool_data_table) + for (auto&& tool_candidate : tool_data_table) { - if (d.tool == toolname && d.os == os) + if (tool_candidate.tool == toolname && tool_candidate.os == os) { - // if the tool data record declares an architecture matching the one we're looking for, that's an exact - // match and we've found the right one - if (auto d_arch = d.arch.get()) + if (!tool_candidate.arch) { - if (*d_arch == arch) + if (!default_tool) { - return &d; + default_tool = &tool_candidate; } } - else + else if (arch == *tool_candidate.arch.get()) { - // if the tool data record does not declare an architecture, we merely record a 'default tool' in - // case a more specific entry naming an architecture is seen - if (default_tool) - { - Checks::unreachable(VCPKG_LINE_INFO, - "there should be only one tool entry with an undeclared architecture"); - } - - default_tool = &d; + return &tool_candidate; } } } - - if (default_tool) - { - return default_tool; - } - - return nullptr; + return default_tool; } - static Optional get_tool_data(StringView tool) + static Optional get_tool_data(const std::vector& tool_data_table, StringView tool) { auto hp = get_host_processor(); #if defined(_WIN32) - auto data = get_raw_tool_data(tool, hp, "windows"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, "windows"); #elif defined(__APPLE__) - auto data = get_raw_tool_data(tool, hp, "osx"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, "osx"); #elif defined(__linux__) - auto data = get_raw_tool_data(tool, hp, "linux"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, "linux"); #elif defined(__FreeBSD__) - auto data = get_raw_tool_data(tool, hp, "freebsd"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, "freebsd"); #elif defined(__OpenBSD__) - auto data = get_raw_tool_data(tool, hp, "openbsd"); + auto data = get_raw_tool_data(tool_data_table, tool, hp, "openbsd"); #else return nullopt; #endif if (!data) { + // Make get_raw_tool_data return ExpectedL and handle error here? return nullopt; } const Optional> version = parse_tool_version_string(data->version); @@ -520,11 +177,11 @@ namespace vcpkg return ToolData{tool.to_string(), *version.get(), data->exeRelativePath, - data->url.to_string(), + data->url, download_subpath, !data->archiveName.empty(), tool_dir_name, - data->sha512.to_string()}; + data->sha512}; } struct PathAndVersion @@ -1044,19 +701,23 @@ namespace vcpkg const Filesystem& fs; const std::shared_ptr downloader; const Path downloads; + const Path config_path; const Path tools; const RequireExactVersions abiToolVersionHandling; vcpkg::Cache path_version_cache; + vcpkg::Lazy> m_tool_data_cache; ToolCacheImpl(const Filesystem& fs, const std::shared_ptr& downloader, Path downloads, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) : fs(fs) , downloader(downloader) , downloads(std::move(downloads)) + , config_path(std::move(config_path)) , tools(std::move(tools)) , abiToolVersionHandling(abiToolVersionHandling) { @@ -1150,7 +811,7 @@ namespace vcpkg get_environment_variable(EnvironmentVariableVcpkgForceSystemBinaries).has_value(); const bool env_force_download_binaries = get_environment_variable(EnvironmentVariableVcpkgForceDownloadedBinaries).has_value(); - const auto maybe_tool_data = get_tool_data(tool.tool_data_name()); + const auto maybe_tool_data = get_tool_data(load_tool_data(), tool.tool_data_name()); const bool download_available = maybe_tool_data.has_value() && !maybe_tool_data.get()->url.empty(); // search for system searchable tools unless forcing downloads and download available const auto system_exe_stems = tool.system_exe_stems(); @@ -1168,7 +829,7 @@ namespace vcpkg if (auto tool_data = maybe_tool_data.get()) { - // If there is an entry for the tool in tools.cpp, use that version as the minimum + // If there is an entry for the tool in vcpkg-tools.json, use that version as the minimum min_version = tool_data->version; if (consider_downloads) @@ -1309,6 +970,18 @@ namespace vcpkg { return get_tool_pathversion(tool, status_sink).version; } + + std::vector load_tool_data() const + { + return m_tool_data_cache.get_lazy([&]() { + auto maybe_tool_data = parse_tool_data(fs, config_path); + if (auto tool_data = maybe_tool_data.get()) + { + return std::move(*tool_data); + } + Checks::msg_exit_with_error(VCPKG_LINE_INFO, maybe_tool_data.error()); + }); + } }; ExpectedL find_system_tar(const ReadOnlyFilesystem& fs) @@ -1381,9 +1054,64 @@ namespace vcpkg std::unique_ptr get_tool_cache(const Filesystem& fs, std::shared_ptr downloader, Path downloads, + Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) { - return std::make_unique(fs, std::move(downloader), downloads, tools, abiToolVersionHandling); + return std::make_unique( + fs, std::move(downloader), downloads, config_path, tools, abiToolVersionHandling); + } + + LocalizedString ToolDataDeserializer::type_name() const { return msg::format(msgAConfigurationObject); } + + Optional ToolDataDeserializer::visit_object(Json::Reader& r, const Json::Object& obj) const + { + static const std::map arch_map{ + {"x86", CPUArchitecture::X86}, + {"x64", CPUArchitecture::X64}, + {"arm", CPUArchitecture::ARM}, + {"arm64", CPUArchitecture::ARM64}, + {"arm64ec", CPUArchitecture::ARM64EC}, + {"s390x", CPUArchitecture::S390X}, + {"ppc64le", CPUArchitecture::PPC64LE}, + {"riscv32", CPUArchitecture::RISCV32}, + {"riscv64", CPUArchitecture::RISCV64}, + {"loongarch32", CPUArchitecture::LOONGARCH32}, + {"loongarch64", CPUArchitecture::LOONGARCH64}, + {"mips64", CPUArchitecture::MIPS64}, + }; + + ArchToolData value; + + r.required_object_field(type_name(), obj, "name", value.tool, Json::UntypedStringDeserializer::instance); + r.required_object_field(type_name(), obj, "os", value.os, Json::UntypedStringDeserializer::instance); + r.required_object_field(type_name(), obj, "version", value.version, Json::UntypedStringDeserializer::instance); + + std::string arch_str; + if (r.optional_object_field(obj, "arch", arch_str, Json::ArchitectureDeserializer::instance)) + { + auto it = arch_map.find(arch_str); + if (it != arch_map.end()) + { + value.arch = it->second; + } + else + { + // TODO: Add an error message + // r.add_generic_error(type_name(), msg::format(msgInvalidArch, msg::arch = arch_str)); + } + } + r.optional_object_field(obj, "executable", value.exeRelativePath, Json::UntypedStringDeserializer::instance); + r.optional_object_field(obj, "url", value.url, Json::UntypedStringDeserializer::instance); + r.optional_object_field(obj, "sha512", value.sha512, Json::Sha512Deserializer::instance); + r.optional_object_field(obj, "archive", value.archiveName, Json::UntypedStringDeserializer::instance); + + // arch string needs to be converted to CPUArchitecture + return value; } -} + + const ToolDataDeserializer ToolDataDeserializer::instance; + + LocalizedString ToolDataArrayDeserializer::type_name() const { return ToolDataDeserializer::instance.type_name(); } + const ToolDataArrayDeserializer ToolDataArrayDeserializer::instance; +} \ No newline at end of file diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 781fd615c9..114f2088e8 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -553,6 +553,7 @@ namespace vcpkg , m_tool_cache(get_tool_cache(fs, m_download_manager, downloads, + scripts / "vcpkg-tools.json", tools, args.exact_abi_tools_versions.value_or(false) ? RequireExactVersions::YES : RequireExactVersions::NO))