diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 1d01412ee051..0e8134bd19ba 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -116,16 +116,41 @@ public HashSet Restore() HashSet? explicitFeeds = null; HashSet? allFeeds = null; + HashSet? reachableFeeds = []; try { - if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds, out allFeeds)) + if (checkNugetFeedResponsiveness) { - // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); - return unresponsiveMissingPackageLocation is null - ? [] - : [unresponsiveMissingPackageLocation]; + // Find feeds that are configured in NuGet.config files and divide them into ones that + // are explicitly configured for the project, and "all feeds" (including inherited ones) + // from other locations on the host outside of the working directory. + (explicitFeeds, allFeeds) = GetAllFeeds(); + var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); + + // If private package registries are configured for C#, then consider those + // in addition to the ones that are configured in `nuget.config` files. + this.dependabotProxy?.RegistryURLs.ForEach(url => explicitFeeds.Add(url)); + + var (explicitFeedsReachable, reachableExplicitFeeds) = + this.CheckSpecifiedFeeds(explicitFeeds); + reachableFeeds.UnionWith(reachableExplicitFeeds); + + reachableFeeds.UnionWith(this.GetReachableNuGetFeeds(inheritedFeeds, isFallback: false)); + + if (inheritedFeeds.Count > 0) + { + compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); + } + + if (!explicitFeedsReachable) + { + // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. + var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); + return unresponsiveMissingPackageLocation is null + ? [] + : [unresponsiveMissingPackageLocation]; + } } using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger)) @@ -167,9 +192,10 @@ public HashSet Restore() logger.LogError($"Failed to restore NuGet packages with nuget.exe: {exc.Message}"); } + // Restore project dependencies with `dotnet restore`. var restoredProjects = RestoreSolutions(out var container); var projects = fileProvider.Projects.Except(restoredProjects); - RestoreProjects(projects, allFeeds, out var containers); + RestoreProjects(projects, reachableFeeds, out var containers); var dependencies = containers.Flatten(container); @@ -192,6 +218,34 @@ public HashSet Restore() return assemblyLookupLocations; } + /// + /// Tests which of the feeds given by are reachable. + /// + /// The feeds to check. + /// Whether the feeds are fallback feeds or not. + /// The list of feeds that could be reached. + private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool isFallback) + { + var fallbackStr = isFallback ? "fallback " : ""; + logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}"); + + var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback); + var reachableFeeds = feedsToCheck + .Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)) + .ToList(); + + if (reachableFeeds.Count == 0) + { + logger.LogWarning($"No {fallbackStr}NuGet feeds are reachable."); + } + else + { + logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}"); + } + + return reachableFeeds; + } + private List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) { var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); @@ -212,21 +266,7 @@ private List GetReachableFallbackNugetFeeds(HashSet? feedsFromNu } } - logger.LogInfo($"Checking fallback NuGet feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: true); - var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)).ToList(); - if (reachableFallbackFeeds.Count == 0) - { - logger.LogWarning("No fallback NuGet feeds are reachable."); - } - else - { - logger.LogInfo($"Reachable fallback NuGet feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}"); - } - - compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); - - return reachableFallbackFeeds; + return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true); } /// @@ -719,53 +759,50 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, } /// - /// Checks that we can connect to all NuGet feeds that are explicitly configured in configuration files - /// as well as any private package registry feeds that are configured. + /// Retrieves a list of excluded NuGet feeds from the corresponding environment variable. /// - /// Outputs the set of explicit feeds. - /// Outputs the set of all feeds (explicit and inherited). - /// True if all feeds are reachable or false otherwise. - private bool CheckFeeds(out HashSet explicitFeeds, out HashSet allFeeds) + private HashSet GetExcludedFeeds() { - (explicitFeeds, allFeeds) = GetAllFeeds(); - HashSet feedsToCheck = explicitFeeds; - - // If private package registries are configured for C#, then check those - // in addition to the ones that are configured in `nuget.config` files. - this.dependabotProxy?.RegistryURLs.ForEach(url => feedsToCheck.Add(url)); - - var allFeedsReachable = this.CheckSpecifiedFeeds(feedsToCheck); + var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) + .ToHashSet(); - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); - if (inheritedFeeds.Count > 0) + if (excludedFeeds.Count > 0) { - logger.LogInfo($"Inherited NuGet feeds (not checked for reachability): {string.Join(", ", inheritedFeeds.OrderBy(f => f))}"); - compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); + logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); } - return allFeedsReachable; + return excludedFeeds; } /// /// Checks that we can connect to the specified NuGet feeds. /// /// The set of package feeds to check. - /// True if all feeds are reachable or false otherwise. - private bool CheckSpecifiedFeeds(HashSet feeds) + /// + /// True if all feeds are reachable or false otherwise. + /// Also returns the list of reachable feeds. + /// + private (bool, List) CheckSpecifiedFeeds(HashSet feeds) { - logger.LogInfo("Checking that NuGet feeds are reachable..."); + // Exclude any feeds that are configured by the corresponding environment variable. + var excludedFeeds = GetExcludedFeeds(); - var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) - .ToHashSet(); + var feedsToCheck = feeds.Where(feed => !excludedFeeds.Contains(feed)).ToHashSet(); + var reachableFeeds = this.GetReachableNuGetFeeds(feedsToCheck, isFallback: false); + var allFeedsReachable = reachableFeeds.Count == feedsToCheck.Count; - if (excludedFeeds.Count > 0) - { - logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); - } + this.EmitUnreachableFeedsDiagnostics(allFeedsReachable); - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); + return (allFeedsReachable, reachableFeeds); + } - var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount)); + /// + /// If is `false`, logs this and emits a diagnostic. + /// Adds a `CompilationInfos` entry either way. + /// + /// Whether all feeds were reachable or not. + private void EmitUnreachableFeedsDiagnostics(bool allFeedsReachable) + { if (!allFeedsReachable) { logger.LogWarning("Found unreachable NuGet feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis."); @@ -779,8 +816,6 @@ private bool CheckSpecifiedFeeds(HashSet feeds) )); } compilationInfoContainer.CompilationInfos.Add(("All NuGet feeds reachable", allFeedsReachable ? "1" : "0")); - - return allFeedsReachable; } private IEnumerable GetFeeds(Func> getNugetFeeds)