diff --git a/src/code/ContainerRegistryServerAPICalls.cs b/src/code/ContainerRegistryServerAPICalls.cs index cd8c4c8be..20232b4f9 100644 --- a/src/code/ContainerRegistryServerAPICalls.cs +++ b/src/code/ContainerRegistryServerAPICalls.cs @@ -91,12 +91,29 @@ public ContainerRegistryServerAPICalls(PSRepositoryInfo repository, PSCmdlet cmd public override Task FindVersionAsync(string packageName, string version, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindVersionAsync()"); - FindResults findResponse = FindVersion(packageName, version, type, out ErrorRecord errRecord); + if (!NuGetVersion.TryParse(version, out NuGetVersion requiredVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + new ArgumentException($"Version {version} to be found is not a valid NuGet version."), + "FindNameFailure", + ErrorCategory.InvalidArgument, + this)); + + return Task.FromResult(emptyResponseResults); + } + + debugMsgs.Enqueue($"'{packageName}' version parsed as '{requiredVersion}'"); + bool includePrereleaseVersions = requiredVersion.IsPrerelease; + + // for FindVersion(), need to consider the specific required version (hence VersionType.SpecificVersion and no version range) + Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.SpecificVersion, versionRange: VersionRange.None, requiredVersion: requiredVersion, includePrereleaseVersions, getOnlyLatest: false, out ErrorRecord errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); + return Task.FromResult(emptyResponseResults); } + FindResults findResponse = new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResult.ToArray(), responseType: containerRegistryFindResponseType); return Task.FromResult(findResponse); } @@ -109,12 +126,16 @@ public override Task FindVersionAsync(string packageName, string ve public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindVersionGlobbingAsync()"); - FindResults findResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord); + + // for FindVersionGlobbing(), need to consider all versions that match version range criteria (hence VersionType.VersionRange and no requiredVersion) + Hashtable[] pkgResults = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: versionRange, requiredVersion: null, includePrerelease, getOnlyLatest: false, out ErrorRecord errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); + return Task.FromResult(emptyResponseResults); } + FindResults findResponse = new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResults.ToArray(), responseType: containerRegistryFindResponseType); return Task.FromResult(findResponse); } @@ -172,9 +193,14 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include public override FindResults FindName(string packageName, bool includePrerelease, ResourceType type, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindName()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); // for FindName(), need to consider all versions (hence VersionType.VersionRange and VersionRange.All, and no requiredVersion) but only pick latest (hence getOnlyLatest: true) - Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: VersionRange.All, requiredVersion: null, includePrerelease, getOnlyLatest: true, out errRecord); + Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: VersionRange.All, requiredVersion: null, includePrerelease, getOnlyLatest: true, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyResponseResults; @@ -192,12 +218,16 @@ public override FindResults FindName(string packageName, bool includePrerelease, public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In ContainerRegistryServerAPICalls::FindNameAsync()"); - FindResults findResponse = FindName(packageName, includePrerelease, type, out ErrorRecord errRecord); + + // for FindName(), need to consider all versions (hence VersionType.VersionRange and VersionRange.All, and no requiredVersion) but only pick latest (hence getOnlyLatest: true) + Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: VersionRange.All, requiredVersion: null, includePrerelease, getOnlyLatest: true, out ErrorRecord errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); + return Task.FromResult(emptyResponseResults); } + FindResults findResponse = new FindResults(stringResponse: new string[] { }, hashtableResponse: pkgResult.ToArray(), responseType: containerRegistryFindResponseType); return Task.FromResult(findResponse); } @@ -265,9 +295,14 @@ public override FindResults FindNameGlobbingWithTag(string packageName, string[] public override FindResults FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindVersionGlobbing()"); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); // for FindVersionGlobbing(), need to consider all versions that match version range criteria (hence VersionType.VersionRange and no requiredVersion) - Hashtable[] pkgResults = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: versionRange, requiredVersion: null, includePrerelease, getOnlyLatest: false, out errRecord); + Hashtable[] pkgResults = FindPackagesWithVersionHelper(packageName, VersionType.VersionRange, versionRange: versionRange, requiredVersion: null, includePrerelease, getOnlyLatest: false, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyResponseResults; @@ -298,9 +333,14 @@ public override FindResults FindVersion(string packageName, string version, Reso _cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{requiredVersion}'"); bool includePrereleaseVersions = requiredVersion.IsPrerelease; + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); // for FindVersion(), need to consider the specific required version (hence VersionType.SpecificVersion and no version range) - Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.SpecificVersion, versionRange: VersionRange.None, requiredVersion: requiredVersion, includePrereleaseVersions, getOnlyLatest: false, out errRecord); + Hashtable[] pkgResult = FindPackagesWithVersionHelper(packageName, VersionType.SpecificVersion, versionRange: VersionRange.None, requiredVersion: requiredVersion, includePrereleaseVersions, getOnlyLatest: false, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyResponseResults; @@ -399,6 +439,10 @@ private Stream InstallVersion( string packageNameLowercase = packageName.ToLower(); string accessToken = string.Empty; string tenantID = string.Empty; + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); try { @@ -415,7 +459,8 @@ private Stream InstallVersion( return null; } - string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out errRecord); + string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return null; @@ -482,7 +527,7 @@ private Stream InstallVersionAsync( return null; } - string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out ErrorRecord errRecord); + string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out ErrorRecord errRecord, errorMsgs, warningMsgs: null, debugMsgs, verboseMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); @@ -490,13 +535,13 @@ private Stream InstallVersionAsync( } verboseMsgs.Enqueue($"Getting manifest for {packageNameLowercase} - {packageVersion}"); - var manifest = GetContainerRegistryRepositoryManifest(packageNameLowercase, packageVersion, containerRegistryAccessToken, out errRecord); + var manifest = GetContainerRegistryRepositoryManifest(packageNameLowercase, packageVersion, containerRegistryAccessToken, out errRecord, debugMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); return null; } - string digest = GetDigestFromManifest(manifest, out errRecord); + string digest = GetDigestFromManifest(manifest, out errRecord, debugMsgs); if (errRecord != null) { errorMsgs.Enqueue(errRecord); @@ -507,7 +552,7 @@ private Stream InstallVersionAsync( HttpContent responseContent; try { - responseContent = GetContainerRegistryBlobAsync(packageNameLowercase, digest, containerRegistryAccessToken).Result; + responseContent = GetContainerRegistryBlobAsync(packageNameLowercase, digest, containerRegistryAccessToken, debugMsgs).Result; } catch (Exception e) { @@ -533,9 +578,9 @@ private Stream InstallVersionAsync( /// If no credential provided at registration then, check if the ACR endpoint can be accessed without a token. If not, try using Azure.Identity to get the az access token, then ACR refresh token and then ACR access token. /// Note: Access token can be empty if the repository is unauthenticated /// - internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isPushOperation, out ErrorRecord errRecord) + internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isPushOperation, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryAccessToken()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryAccessToken()"); string accessToken = string.Empty; string containerRegistryAccessToken = string.Empty; string tenantID = string.Empty; @@ -543,7 +588,7 @@ internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isP if (!string.IsNullOrEmpty(_cachedContainterRegistryToken)) { - _cmdletPassedIn.WriteVerbose("Using cached container registry access token."); + verboseMsgs?.Enqueue("Using cached container registry access token."); return _cachedContainterRegistryToken; } @@ -555,17 +600,17 @@ internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isP repositoryCredentialInfo, _cmdletPassedIn); - _cmdletPassedIn.WriteVerbose("Access token retrieved."); + verboseMsgs?.Enqueue("Access token retrieved."); tenantID = repositoryCredentialInfo.SecretName; } else { // A container registry repository is determined to be unauthenticated if it allows anonymous pull access. However, push operations always require authentication. - bool isRepositoryUnauthenticated = isPushOperation ? false : IsContainerRegistryUnauthenticated(Repository.Uri.ToString(), needCatalogAccess, out errRecord, out accessToken); - _cmdletPassedIn.WriteInformation($"Value of isRepositoryUnauthenticated: {isRepositoryUnauthenticated}", new string[] { "PSRGContainerRegistryUnauthenticatedCheck" }); + bool isRepositoryUnauthenticated = isPushOperation ? false : IsContainerRegistryUnauthenticated(Repository.Uri.ToString(), needCatalogAccess, out errRecord, out accessToken, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + verboseMsgs?.Enqueue($"Value of isRepositoryUnauthenticated: {isRepositoryUnauthenticated}"); - _cmdletPassedIn.WriteDebug($"Is repository unauthenticated: {isRepositoryUnauthenticated}"); + debugMsgs?.Enqueue($"Is repository unauthenticated: {isRepositoryUnauthenticated}"); if (errRecord != null) { @@ -574,7 +619,7 @@ internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isP if (!string.IsNullOrEmpty(accessToken)) { - _cmdletPassedIn.WriteVerbose("Anonymous access token retrieved."); + verboseMsgs?.Enqueue("Anonymous access token retrieved."); return accessToken; } @@ -594,24 +639,24 @@ internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isP } else { - _cmdletPassedIn.WriteVerbose("Repository is unauthenticated"); + verboseMsgs?.Enqueue("Repository is unauthenticated"); return null; } } - var containerRegistryRefreshToken = GetContainerRegistryRefreshToken(tenantID, accessToken, out errRecord); + var containerRegistryRefreshToken = GetContainerRegistryRefreshToken(tenantID, accessToken, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return null; } - containerRegistryAccessToken = GetContainerRegistryAccessTokenByRefreshToken(containerRegistryRefreshToken, out errRecord); + containerRegistryAccessToken = GetContainerRegistryAccessTokenByRefreshToken(containerRegistryRefreshToken, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return null; } - _cmdletPassedIn.WriteVerbose("Container registry access token retrieved."); + verboseMsgs?.Enqueue("Container registry access token retrieved."); _cachedContainterRegistryToken = containerRegistryAccessToken; return containerRegistryAccessToken; @@ -620,9 +665,9 @@ internal string GetContainerRegistryAccessToken(bool needCatalogAccess, bool isP /// /// Checks if container registry repository is unauthenticated. /// - internal bool IsContainerRegistryUnauthenticated(string containerRegistryUrl, bool needCatalogAccess, out ErrorRecord errRecord, out string anonymousAccessToken) + internal bool IsContainerRegistryUnauthenticated(string containerRegistryUrl, bool needCatalogAccess, out ErrorRecord errRecord, out string anonymousAccessToken, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::IsContainerRegistryUnauthenticated()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::IsContainerRegistryUnauthenticated()"); errRecord = null; anonymousAccessToken = string.Empty; string endpoint = $"{containerRegistryUrl}/v2/"; @@ -664,24 +709,24 @@ internal bool IsContainerRegistryUnauthenticated(string containerRegistryUrl, bo string url = needCatalogAccess ? String.Format(authUrlTemplate, realm, service, catalogScope) : String.Format(authUrlTemplate, realm, service, defaultScope); - _cmdletPassedIn.WriteDebug($"Getting anonymous access token from the realm: {url}"); + debugMsgs?.Enqueue($"Getting anonymous access token from the realm: {url}"); // we don't check the error record here because we want to return false if we get a 401 and not throw an error - _cmdletPassedIn.WriteDebug($"Getting anonymous access token from the realm: {url}"); + debugMsgs?.Enqueue($"Getting anonymous access token from the realm: {url}"); ErrorRecord errRecordTemp = null; - var results = GetHttpResponseJObjectUsingContentHeaders(url, HttpMethod.Get, content, contentHeaders, out errRecordTemp); + var results = GetHttpResponseJObjectUsingContentHeaders(url, HttpMethod.Get, content, contentHeaders, out errRecordTemp, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (results == null) { - _cmdletPassedIn.WriteDebug("Failed to get access token from the realm. results is null."); - _cmdletPassedIn.WriteDebug($"ErrorRecord: {errRecordTemp}"); + debugMsgs?.Enqueue("Failed to get access token from the realm. results is null."); + debugMsgs?.Enqueue($"ErrorRecord: {errRecordTemp}"); return false; } if (results["access_token"] == null) { - _cmdletPassedIn.WriteDebug($"Failed to get access token from the realm. access_token is null. results: {results}"); + debugMsgs?.Enqueue($"Failed to get access token from the realm. access_token is null. results: {results}"); return false; } @@ -719,13 +764,13 @@ internal bool IsContainerRegistryUnauthenticated(string containerRegistryUrl, bo /// /// Given the access token retrieved from credentials, gets the refresh token. /// - internal string GetContainerRegistryRefreshToken(string tenant, string accessToken, out ErrorRecord errRecord) + internal string GetContainerRegistryRefreshToken(string tenant, string accessToken, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryRefreshToken()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryRefreshToken()"); string content = string.Format(containerRegistryRefreshTokenTemplate, Registry, tenant, accessToken); var contentHeaders = new Collection> { new KeyValuePair("Content-Type", "application/x-www-form-urlencoded") }; string exchangeUrl = string.Format(containerRegistryOAuthExchangeUrlTemplate, Registry); - var results = GetHttpResponseJObjectUsingContentHeaders(exchangeUrl, HttpMethod.Post, content, contentHeaders, out errRecord); + var results = GetHttpResponseJObjectUsingContentHeaders(exchangeUrl, HttpMethod.Post, content, contentHeaders, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null || results == null || results["refresh_token"] == null) { return string.Empty; @@ -737,13 +782,13 @@ internal string GetContainerRegistryRefreshToken(string tenant, string accessTok /// /// Given the refresh token, gets the new access token with appropriate scope access permissions. /// - internal string GetContainerRegistryAccessTokenByRefreshToken(string refreshToken, out ErrorRecord errRecord) + internal string GetContainerRegistryAccessTokenByRefreshToken(string refreshToken, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryAccessTokenByRefreshToken()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryAccessTokenByRefreshToken()"); string content = string.Format(containerRegistryAccessTokenTemplate, Registry, refreshToken); var contentHeaders = new Collection> { new KeyValuePair("Content-Type", "application/x-www-form-urlencoded") }; string tokenUrl = string.Format(containerRegistryOAuthTokenUrlTemplate, Registry); - var results = GetHttpResponseJObjectUsingContentHeaders(tokenUrl, HttpMethod.Post, content, contentHeaders, out errRecord); + var results = GetHttpResponseJObjectUsingContentHeaders(tokenUrl, HttpMethod.Post, content, contentHeaders, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null || results == null || results["access_token"] == null) { return string.Empty; @@ -759,9 +804,9 @@ internal string GetContainerRegistryAccessTokenByRefreshToken(string refreshToke /// /// Parses package manifest JObject to find digest entry, which is the SHA needed to identify and get the package. /// - private string GetDigestFromManifest(JObject manifest, out ErrorRecord errRecord) + private string GetDigestFromManifest(JObject manifest, out ErrorRecord errRecord, ConcurrentQueue debugMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetDigestFromManifest()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetDigestFromManifest()"); errRecord = null; string digest = String.Empty; @@ -803,32 +848,32 @@ private string GetDigestFromManifest(JObject manifest, out ErrorRecord errRecord /// /// Gets the manifest for a package (ie repository in container registry terms) from the repository (ie registry in container registry terms) /// - internal JObject GetContainerRegistryRepositoryManifest(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord) + internal JObject GetContainerRegistryRepositoryManifest(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord, ConcurrentQueue debugMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryRepositoryManifest()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryRepositoryManifest()"); // example of manifestUrl: https://psgetregistry.azurecr.io/hello-world:3.0.0 string manifestUrl = string.Format(containerRegistryManifestUrlTemplate, Registry, packageName, version); var defaultHeaders = GetDefaultHeaders(containerRegistryAccessToken); - return GetHttpResponseJObjectUsingDefaultHeaders(manifestUrl, HttpMethod.Get, defaultHeaders, out errRecord); + return GetHttpResponseJObjectUsingDefaultHeaders(manifestUrl, HttpMethod.Get, defaultHeaders, out errRecord, debugMsgs: debugMsgs); } /// /// Get the blob for the package (ie repository in container registry terms) from the repository (ie registry in container registry terms) /// Used when installing the package /// - internal async Task GetContainerRegistryBlobAsync(string packageName, string digest, string containerRegistryAccessToken) + internal async Task GetContainerRegistryBlobAsync(string packageName, string digest, string containerRegistryAccessToken, ConcurrentQueue debugMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryBlobAsync()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryBlobAsync()"); string blobUrl = string.Format(containerRegistryBlobDownloadUrlTemplate, Registry, packageName, digest); var defaultHeaders = GetDefaultHeaders(containerRegistryAccessToken); - return await GetHttpContentResponseJObject(blobUrl, defaultHeaders); + return await GetHttpContentResponseJObject(blobUrl, defaultHeaders, debugMsgs); } /// /// Gets the image tags associated with the package (i.e repository in container registry terms), where the tag corresponds to the package's versions. /// If the package version is specified search for that specific tag for the image, if the package version is "*" search for all tags for the image. /// - internal JObject FindContainerRegistryImageTags(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord) + internal JObject FindContainerRegistryImageTags(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { /* { @@ -840,11 +885,11 @@ internal JObject FindContainerRegistryImageTags(string packageName, string versi ] } */ - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindContainerRegistryImageTags()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::FindContainerRegistryImageTags()"); string resolvedVersion = string.Equals(version, "*", StringComparison.OrdinalIgnoreCase) ? null : $"/{version}"; string findImageUrl = string.Format(containerRegistryFindImageVersionUrlTemplate, Registry, packageName); var defaultHeaders = GetDefaultHeaders(containerRegistryAccessToken); - return GetHttpResponseJObjectUsingDefaultHeaders(findImageUrl, HttpMethod.Get, defaultHeaders, out errRecord); + return GetHttpResponseJObjectUsingDefaultHeaders(findImageUrl, HttpMethod.Get, defaultHeaders, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } /// @@ -864,12 +909,12 @@ internal JObject FindAllRepositories(string containerRegistryAccessToken, out Er /// /// Get metadata for a package version. /// - internal Hashtable GetContainerRegistryMetadata(string packageName, string exactTagVersion, string containerRegistryAccessToken, out ErrorRecord errRecord) + internal Hashtable GetContainerRegistryMetadata(string packageName, string exactTagVersion, string containerRegistryAccessToken, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetContainerRegistryMetadata()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetContainerRegistryMetadata()"); Hashtable requiredVersionResponse = new(); - JObject foundTags = FindContainerRegistryManifest(packageName, exactTagVersion, containerRegistryAccessToken, out errRecord); + JObject foundTags = FindContainerRegistryManifest(packageName, exactTagVersion, containerRegistryAccessToken, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return requiredVersionResponse; @@ -898,7 +943,7 @@ internal Hashtable GetContainerRegistryMetadata(string packageName, string exact } */ - ContainerRegistryInfo serverPkgInfo = GetMetadataProperty(foundTags, packageName, out errRecord); + ContainerRegistryInfo serverPkgInfo = GetMetadataProperty(foundTags, packageName, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return requiredVersionResponse; @@ -959,7 +1004,7 @@ internal Hashtable GetContainerRegistryMetadata(string packageName, string exact return requiredVersionResponse; } - _cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{pkgVersion}'"); + debugMsgs?.Enqueue($"'{packageName}' version parsed as '{pkgVersion}'"); if (pkgVersion.ToNormalizedString() == requiredVersion.ToNormalizedString()) { requiredVersionResponse = serverPkgInfo.ToHashtable(); @@ -983,22 +1028,22 @@ internal Hashtable GetContainerRegistryMetadata(string packageName, string exact /// /// Get the manifest associated with the package version. /// - internal JObject FindContainerRegistryManifest(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord) + internal JObject FindContainerRegistryManifest(string packageName, string version, string containerRegistryAccessToken, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindContainerRegistryManifest()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::FindContainerRegistryManifest()"); var createManifestUrl = string.Format(containerRegistryManifestUrlTemplate, Registry, packageName, version); - _cmdletPassedIn.WriteDebug($"GET manifest url: {createManifestUrl}"); + debugMsgs?.Enqueue($"GET manifest url: {createManifestUrl}"); var defaultHeaders = GetDefaultHeaders(containerRegistryAccessToken); - return GetHttpResponseJObjectUsingDefaultHeaders(createManifestUrl, HttpMethod.Get, defaultHeaders, out errRecord); + return GetHttpResponseJObjectUsingDefaultHeaders(createManifestUrl, HttpMethod.Get, defaultHeaders, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } /// /// Get metadata for the package by parsing its manifest. /// - internal ContainerRegistryInfo GetMetadataProperty(JObject foundTags, string packageName, out ErrorRecord errRecord) + internal ContainerRegistryInfo GetMetadataProperty(JObject foundTags, string packageName, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetMetadataProperty()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetMetadataProperty()"); errRecord = null; ContainerRegistryInfo serverPkgInfo = null; @@ -1091,9 +1136,9 @@ internal async Task UploadManifest(string packageName, stri } } - internal async Task GetHttpContentResponseJObject(string url, Collection> defaultHeaders) + internal async Task GetHttpContentResponseJObject(string url, Collection> defaultHeaders, ConcurrentQueue debugMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetHttpContentResponseJObject()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetHttpContentResponseJObject()"); try { HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url); @@ -1109,9 +1154,9 @@ internal async Task GetHttpContentResponseJObject(string url, Colle /// /// Get response object when using default headers in the request. /// - internal JObject GetHttpResponseJObjectUsingDefaultHeaders(string url, HttpMethod method, Collection> defaultHeaders, out ErrorRecord errRecord, bool usePagination = false) + internal JObject GetHttpResponseJObjectUsingDefaultHeaders(string url, HttpMethod method, Collection> defaultHeaders, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null, bool usePagination = false) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetHttpResponseJObjectUsingDefaultHeaders()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetHttpResponseJObjectUsingDefaultHeaders()"); try { errRecord = null; @@ -1160,9 +1205,9 @@ internal JObject GetHttpResponseJObjectUsingDefaultHeaders(string url, HttpMetho /// /// Get response object when using content headers in the request. /// - internal JObject GetHttpResponseJObjectUsingContentHeaders(string url, HttpMethod method, string content, Collection> contentHeaders, out ErrorRecord errRecord) + internal JObject GetHttpResponseJObjectUsingContentHeaders(string url, HttpMethod method, string content, Collection> contentHeaders, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetHttpResponseJObjectUsingContentHeaders()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetHttpResponseJObjectUsingContentHeaders()"); errRecord = null; try { @@ -1486,7 +1531,12 @@ internal bool PushNupkgContainerRegistry( // Get access token (includes refresh tokens) _cmdletPassedIn.WriteVerbose($"Get access token for container registry server."); - var containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: true, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + var containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: true, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return false; @@ -1942,22 +1992,22 @@ internal async Task EndUploadBlob(string location, string f /// /// Helper method for find scenarios. /// - private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionType versionType, VersionRange versionRange, NuGetVersion requiredVersion, bool includePrerelease, bool getOnlyLatest, out ErrorRecord errRecord) + private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionType versionType, VersionRange versionRange, NuGetVersion requiredVersion, bool includePrerelease, bool getOnlyLatest, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindPackagesWithVersionHelper()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::FindPackagesWithVersionHelper()"); string accessToken = string.Empty; string tenantID = string.Empty; string registryUrl = Repository.Uri.ToString(); string packageNameLowercase = packageName.ToLower(); string packageNameForFind = PrependMARPrefix(packageNameLowercase); - string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false,out errRecord); + string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: false, isPushOperation: false, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyHashResponses; } - var foundTags = FindContainerRegistryImageTags(packageNameForFind, "*", containerRegistryAccessToken, out errRecord); + var foundTags = FindContainerRegistryImageTags(packageNameForFind, "*", containerRegistryAccessToken, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null || foundTags == null) { return emptyHashResponses; @@ -1966,10 +2016,10 @@ private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionTyp List latestVersionResponse = new List(); List allVersionsList = foundTags["tags"].ToList(); - SortedDictionary sortedQualifyingPkgs = GetPackagesWithRequiredVersion(allVersionsList, versionType, versionRange, requiredVersion, packageNameForFind, includePrerelease, out errRecord); + SortedDictionary sortedQualifyingPkgs = GetPackagesWithRequiredVersion(allVersionsList, versionType, versionRange, requiredVersion, packageNameForFind, includePrerelease, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null && sortedQualifyingPkgs?.Count == 0) { - _cmdletPassedIn.WriteDebug("No qualifying packages found for the specified criteria."); + debugMsgs?.Enqueue("No qualifying packages found for the specified criteria."); return emptyHashResponses; } @@ -1978,7 +2028,7 @@ private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionTyp foreach (var pkgVersionTag in pkgsInDescendingOrder) { string exactTagVersion = pkgVersionTag.Value.ToString(); - Hashtable metadata = GetContainerRegistryMetadata(packageNameForFind, exactTagVersion, containerRegistryAccessToken, out errRecord); + Hashtable metadata = GetContainerRegistryMetadata(packageNameForFind, exactTagVersion, containerRegistryAccessToken, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null || metadata.Count == 0) { return emptyHashResponses; @@ -1998,9 +2048,9 @@ private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionTyp /// /// Helper method used for find scenarios that resolves versions required from all versions found. /// - private SortedDictionary GetPackagesWithRequiredVersion(List allPkgVersions, VersionType versionType, VersionRange versionRange, NuGetVersion specificVersion, string packageName, bool includePrerelease, out ErrorRecord errRecord) + private SortedDictionary GetPackagesWithRequiredVersion(List allPkgVersions, VersionType versionType, VersionRange versionRange, NuGetVersion specificVersion, string packageName, bool includePrerelease, out ErrorRecord errRecord, ConcurrentQueue errorMsgs = null, ConcurrentQueue warningMsgs = null, ConcurrentQueue debugMsgs = null, ConcurrentQueue verboseMsgs = null) { - _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::GetPackagesWithRequiredVersion()"); + debugMsgs?.Enqueue("In ContainerRegistryServerAPICalls::GetPackagesWithRequiredVersion()"); errRecord = null; // we need NuGetVersion to sort versions by order, and string pkgVersionString (which is the exact tag from the server) to call GetContainerRegistryMetadata() later with exact version tag. SortedDictionary sortedPkgs = new SortedDictionary(VersionComparer.Default); @@ -2018,12 +2068,12 @@ private Hashtable[] FindPackagesWithVersionHelper(string packageName, VersionTyp ErrorCategory.InvalidArgument, this); - _cmdletPassedIn.WriteError(errRecord); - _cmdletPassedIn.WriteDebug($"Skipping package '{packageName}' with version '{pkgVersionString}' as it is not a valid NuGet version."); + errorMsgs?.Enqueue(errRecord); + debugMsgs?.Enqueue($"Skipping package '{packageName}' with version '{pkgVersionString}' as it is not a valid NuGet version."); continue; // skip this version and continue with the next one } - _cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{pkgVersion}'"); + debugMsgs?.Enqueue($"'{packageName}' version parsed as '{pkgVersion}'"); if (isSpecificVersionSearch) { @@ -2063,7 +2113,12 @@ private FindResults FindPackages(string packageName, bool includePrerelease, out { _cmdletPassedIn.WriteDebug("In ContainerRegistryServerAPICalls::FindPackages()"); errRecord = null; - string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: true, isPushOperation: false, out errRecord); + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + string containerRegistryAccessToken = GetContainerRegistryAccessToken(needCatalogAccess: true, isPushOperation: false, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyResponseResults; @@ -2100,7 +2155,8 @@ private FindResults FindPackages(string packageName, bool includePrerelease, out _cmdletPassedIn.WriteDebug($"Found repository: {repositoryName}"); - repositoriesList.AddRange(FindPackagesWithVersionHelper(repositoryName, VersionType.VersionRange, versionRange: VersionRange.All, requiredVersion: null, includePrerelease, getOnlyLatest: true, out errRecord)); + repositoriesList.AddRange(FindPackagesWithVersionHelper(repositoryName, VersionType.VersionRange, versionRange: VersionRange.All, requiredVersion: null, includePrerelease, getOnlyLatest: true, out errRecord, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); if (errRecord != null) { return emptyResponseResults; diff --git a/src/code/FindHelper.cs b/src/code/FindHelper.cs index 06a0e9df0..5e773fc11 100644 --- a/src/code/FindHelper.cs +++ b/src/code/FindHelper.cs @@ -46,10 +46,6 @@ internal class FindHelper // If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies. private ConcurrentDictionary> _packagesFound; - // Creates a new instance of depPkgsFound each time FindDependencyPackages() is called. - // This will eventually return the PSResourceInfo object to the main cmdlet class. - private ConcurrentDictionary depPkgsFound; - // Contains the latest found version of a particular package. private ConcurrentDictionary _knownLatestPkgVersion; @@ -1060,13 +1056,36 @@ private IEnumerable SearchByNames(ServerApiCall currentServer, R // After retrieving all packages find their dependencies if (_includeDependencies) { - foreach (PSResourceInfo currentPkg in parentPkgs) + // Resolving each parent package's dependency closure is independent work, so do it concurrently. + // yield return cannot be used inside Parallel.ForEach, so collect results into a thread-safe bag first. + ConcurrentBag dependencyPkgs = new ConcurrentBag(); + int processorCount = Environment.ProcessorCount; + int maxDegreeOfParallelism = processorCount * 4; + if (parentPkgs.Count > processorCount) + { + Parallel.ForEach(parentPkgs, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, currentPkg => + { + foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + { + dependencyPkgs.Add(pkgDep); + } + }); + } + else { - foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + foreach (PSResourceInfo currentPkg in parentPkgs) { - yield return pkgDep; + foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository)) + { + dependencyPkgs.Add(pkgDep); + } } } + + foreach (PSResourceInfo pkgDep in dependencyPkgs) + { + yield return pkgDep; + } } } @@ -1158,20 +1177,50 @@ private string FormatPkgVersionString(PSResourceInfo pkg) internal IEnumerable FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) { - depPkgsFound = new ConcurrentDictionary(); - _cmdletPassedIn.WriteDebug($"In FindHelper::FindDependencyPackages() - {currentPkg.Name}"); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository); + // Pipeline-thread callers: collect diagnostics locally and drain them to the cmdlet on this thread. + ConcurrentQueue errorMsgs = new ConcurrentQueue(); + ConcurrentQueue warningMsgs = new ConcurrentQueue(); + ConcurrentQueue debugMsgs = new ConcurrentQueue(); + ConcurrentQueue verboseMsgs = new ConcurrentQueue(); + + var depPkgs = FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + return depPkgs; + } + + // Overload for worker-thread callers: diagnostics are routed to the caller-provided queues and drained by the caller on the pipeline thread. + internal IEnumerable FindDependencyPackages( + ServerApiCall currentServer, + ResponseUtil currentResponseUtil, + PSResourceInfo currentPkg, + PSRepositoryInfo repository, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs) + { + // Use a local instance so multiple parent packages can resolve their dependency closures concurrently + // without racing on shared state. + ConcurrentDictionary depPkgsFound = new ConcurrentDictionary(); + debugMsgs.Enqueue($"In FindHelper::FindDependencyPackages() - {currentPkg.Name}"); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); return depPkgsFound.Values.ToList(); } // Method 2 - internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository) + internal void FindDependencyPackagesHelper( + ServerApiCall currentServer, + ResponseUtil currentResponseUtil, + PSResourceInfo currentPkg, + PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs) { - ConcurrentQueue errorMsgs = new ConcurrentQueue(); - ConcurrentQueue verboseMsgs = new ConcurrentQueue(); - ConcurrentQueue debugMsgs = new ConcurrentQueue(); - ConcurrentQueue warningMsgs = new ConcurrentQueue(); debugMsgs.Enqueue("In FindHelper::FindDependencyPackagesHelper()"); if (currentPkg.Dependencies.Length > 0) @@ -1185,7 +1234,7 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep => { debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'"); - FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); }); } else @@ -1193,11 +1242,9 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response foreach (var dep in currentPkg.Dependencies) { debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'"); - FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } - - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } @@ -1208,6 +1255,7 @@ private void FindDependencyPackageVersion( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1228,7 +1276,7 @@ private void FindDependencyPackageVersion( else { // Find this version from the server - depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } else if (dep.VersionRange.HasLowerBound && dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion)) @@ -1245,7 +1293,7 @@ private void FindDependencyPackageVersion( } else { - depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } else @@ -1261,7 +1309,7 @@ private void FindDependencyPackageVersion( } else { - depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } } @@ -1273,6 +1321,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1351,7 +1400,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } } @@ -1366,6 +1415,7 @@ private PSResourceInfo FindDependencyWithLowerBound( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1419,7 +1469,7 @@ private PSResourceInfo FindDependencyWithLowerBound( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } } @@ -1434,6 +1484,7 @@ private PSResourceInfo FindDependencyWithUpperBound( ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, + ConcurrentDictionary depPkgsFound, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, @@ -1490,7 +1541,7 @@ private PSResourceInfo FindDependencyWithUpperBound( // This will eventually return the PSResourceInfo object to the main cmdlet class. debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found"); depPkgsFound.TryAdd(key, depPkg); - FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository); + FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); } } } diff --git a/src/code/InstallHelper.cs b/src/code/InstallHelper.cs index e3f95b616..4e13b4b5f 100644 --- a/src/code/InstallHelper.cs +++ b/src/code/InstallHelper.cs @@ -516,73 +516,123 @@ private List InstallPackages( FindHelper findHelper) { _cmdletPassedIn.WriteDebug("In InstallHelper::InstallPackages()"); - + List pkgsSuccessfullyInstalled = new(); - // Install parent package to the temp directory, - // Get the dependencies from the installed package, - // Install all dependencies to temp directory. - // If a single dependency fails to install, roll back by deleting the temp directory. + // ---------- Phase 1 (pipeline thread): resolve each parent package and evaluate ShouldProcess ---------- + // ShouldProcess / -WhatIf / -Confirm and Write* must run on the pipeline thread, so all gating happens + // here, before any parallel download work begins. Packages that pass the gate become work items. + List workItems = new(); foreach (var parentPackage in pkgNamesToInstall) { - string tempInstallPath = CreateInstallationTempPath(); - - try + PSResourceInfo pkgToInstall = FindParentPackage( + searchVersionType: _versionType, + specificVersion: _nugetVersion, + versionRange: _versionRange, + pkgNameToInstall: parentPackage, + repository: repository, + currentServer: currentServer, + currentResponseUtil: currentResponseUtil, + pkgVersion: out string pkgVersion, + errRecord: out ErrorRecord findErrRecord); + + if (findErrRecord != null) { - // Hashtable has the key as the package name - // and value as a Hashtable of specific package info: - // packageName, { version = "", isScript = "", isModule = "", pkg = "", etc. } - // Install parent package to the temp directory. - ConcurrentDictionary packagesHash = BeginPackageInstall( - searchVersionType: _versionType, - specificVersion: _nugetVersion, - versionRange: _versionRange, - pkgNameToInstall: parentPackage, - repository: repository, - currentServer: currentServer, - currentResponseUtil: currentResponseUtil, - tempInstallPath: tempInstallPath, - skipDependencyCheck: skipDependencyCheck, - packagesHash: new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase), - warning: out string warning, - errRecord: out ErrorRecord errRecord); - - // At this point all packages are installed to temp path. - if (errRecord != null) + if (findErrRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) { - if (errRecord.FullyQualifiedErrorId.Equals("PackageNotFound")) - { - _cmdletPassedIn.WriteVerbose(errRecord.Exception.Message); - } - else - { - _cmdletPassedIn.WriteError(errRecord); - } - - continue; + _cmdletPassedIn.WriteVerbose(findErrRecord.Exception.Message); } - if (warning != null) + else { - _cmdletPassedIn.WriteWarning(warning); + _cmdletPassedIn.WriteError(findErrRecord); } - if (packagesHash.Count == 0) - { - continue; - } + continue; + } + + if (pkgToInstall == null) + { + continue; + } + + // Check to see if the pkg is already installed (unless -Reinstall was specified). + if (!_reinstall && _packagesOnMachine.Contains($"{pkgToInstall.Name}{pkgVersion}")) + { + _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); + + // Remove from tracking list of packages to install. + _pkgNamesToInstall.RemoveAll(x => x.Equals(pkgToInstall.Name, StringComparison.InvariantCultureIgnoreCase)); + + continue; + } + + // ShouldProcess gate (handles -WhatIf / -Confirm) on the pipeline thread. + string shouldProcessTarget = _savePkg + ? $"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'" + : $"Package to install: '{pkgToInstall.Name}', version: '{pkgVersion}'"; + if (!_cmdletPassedIn.ShouldProcess(shouldProcessTarget)) + { + continue; + } + + workItems.Add(new ParentInstallWorkItem + { + PkgToInstall = pkgToInstall, + PkgVersion = pkgVersion, + TempInstallPath = CreateInstallationTempPath() + }); + } - Hashtable parentPkgInfo = packagesHash[parentPackage] as Hashtable; - PSResourceInfo parentPkgObj = parentPkgInfo["psResourceInfoPkg"] as PSResourceInfo; + if (workItems.Count == 0) + { + return pkgsSuccessfullyInstalled; + } + + // ---------- Phase 2 (parallel): download each parent + its dependencies to its own temp path ---------- + // This is network-bound work. No pipeline-thread calls are made here; all host messages are queued + // per work item and drained in Phase 3. Each work item downloads into its own temp path and its own + // packagesHash, so there is no shared mutable state between parents. + int processorCount = Environment.ProcessorCount; + if (workItems.Count > 1) + { + int maxDegreeOfParallelism = processorCount * 4; + Parallel.ForEach(workItems, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, workItem => + { + workItem.PackagesHash = DownloadParentAndDeps( + workItem.PkgToInstall, workItem.PkgVersion, workItem.TempInstallPath, repository, + currentServer, currentResponseUtil, skipDependencyCheck, + workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs, + out bool succeeded); + workItem.Succeeded = succeeded; + }); + } + else + { + ParentInstallWorkItem workItem = workItems[0]; + workItem.PackagesHash = DownloadParentAndDeps( + workItem.PkgToInstall, workItem.PkgVersion, workItem.TempInstallPath, repository, + currentServer, currentResponseUtil, skipDependencyCheck, + workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs, + out bool succeeded); + workItem.Succeeded = succeeded; + } - // If -WhatIf is passed in, early out. - if (_cmdletPassedIn.MyInvocation.BoundParameters.ContainsKey("WhatIf") && (SwitchParameter)_cmdletPassedIn.MyInvocation.BoundParameters["WhatIf"] == true) + // ---------- Phase 3 (pipeline thread): drain messages, move content to final location, record results ---------- + // If a single dependency fails to install, roll back that parent by deleting its temp directory. + foreach (ParentInstallWorkItem workItem in workItems) + { + try + { + Utils.WriteOutConcurrentQueue(_cmdletPassedIn, workItem.ErrorMsgs, workItem.WarningMsgs, workItem.DebugMsgs, workItem.VerboseMsgs); + + if (!workItem.Succeeded || workItem.PackagesHash == null || workItem.PackagesHash.Count == 0) { - return pkgsSuccessfullyInstalled; + continue; } // Parent package and dependencies are now installed to temp directory. // Try to move all package directories from temp directory to final destination. - if (!TryMoveInstallContent(tempInstallPath, scope, packagesHash)) + if (!TryMoveInstallContent(workItem.TempInstallPath, scope, workItem.PackagesHash)) { _cmdletPassedIn.WriteError(new ErrorRecord( new InvalidOperationException(), @@ -592,9 +642,9 @@ private List InstallPackages( } else { - foreach (string pkgName in packagesHash.Keys) + foreach (string pkgName in workItem.PackagesHash.Keys) { - Hashtable pkgInfo = packagesHash[pkgName] as Hashtable; + Hashtable pkgInfo = workItem.PackagesHash[pkgName] as Hashtable; pkgsSuccessfullyInstalled.Add(pkgInfo["psResourceInfoPkg"] as PSResourceInfo); // Add each pkg to _packagesOnMachine (ie pkgs fully installed on the machine). @@ -610,11 +660,11 @@ private List InstallPackages( ErrorCategory.InvalidOperation, _cmdletPassedIn)); - throw e; + throw; } finally { - DeleteInstallationTempPath(tempInstallPath); + DeleteInstallationTempPath(workItem.TempInstallPath); } } @@ -622,9 +672,27 @@ private List InstallPackages( } /// - /// Installs a single package to the temporary path. + /// Tracks the per-parent state used to parallelize parent-package installation across the three phases. + /// Each parent has its own temp path, result hash, and message queues so there is no shared mutable state + /// during the parallel download phase. + /// + private sealed class ParentInstallWorkItem + { + public PSResourceInfo PkgToInstall; + public string PkgVersion; + public string TempInstallPath; + public ConcurrentDictionary PackagesHash; + public bool Succeeded; + public readonly ConcurrentQueue ErrorMsgs = new(); + public readonly ConcurrentQueue WarningMsgs = new(); + public readonly ConcurrentQueue DebugMsgs = new(); + public readonly ConcurrentQueue VerboseMsgs = new(); + } + + /// + /// Resolves the parent package to install (find + version selection). Must run on the pipeline thread. /// - private ConcurrentDictionary BeginPackageInstall( + private PSResourceInfo FindParentPackage( VersionType searchVersionType, NuGetVersion specificVersion, VersionRange versionRange, @@ -632,15 +700,12 @@ private ConcurrentDictionary BeginPackageInstall( PSRepositoryInfo repository, ServerApiCall currentServer, ResponseUtil currentResponseUtil, - string tempInstallPath, - bool skipDependencyCheck, - ConcurrentDictionary packagesHash, - out string warning, + out string pkgVersion, out ErrorRecord errRecord) { - _cmdletPassedIn.WriteDebug("In InstallHelper::BeginPackageInstall()"); + _cmdletPassedIn.WriteDebug("In InstallHelper::FindParentPackage()"); FindResults responses = null; - warning = null; + pkgVersion = null; errRecord = null; // Find the parent package that needs to be installed @@ -652,7 +717,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findVersionGlobbingErrRecord != null || responses.IsFindResultsEmpty()) { errRecord = findVersionGlobbingErrRecord; - return packagesHash; + return null; } break; @@ -664,7 +729,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findVersionErrRecord != null) { errRecord = findVersionErrRecord; - return packagesHash; + return null; } break; @@ -675,7 +740,7 @@ private ConcurrentDictionary BeginPackageInstall( if (findNameErrRecord != null) { errRecord = findNameErrRecord; - return packagesHash; + return null; } break; @@ -721,11 +786,11 @@ private ConcurrentDictionary BeginPackageInstall( if (pkgToInstall == null) { - return packagesHash; + return null; } pkgToInstall.RepositorySourceLocation = repository.Uri.ToString(); - pkgToInstall.AdditionalMetadata.TryGetValue("NormalizedVersion", out string pkgVersion); + pkgToInstall.AdditionalMetadata.TryGetValue("NormalizedVersion", out pkgVersion); if (pkgVersion == null) { // Not all NuGet providers (e.g. Artifactory, possibly others) send NormalizedVersion in NuGet package responses. @@ -745,117 +810,67 @@ private ConcurrentDictionary BeginPackageInstall( } // Check to see if the pkg is already installed (ie the pkg is installed and the version satisfies the version range provided via param) - // TODO: can use cache for this - if (!_reinstall) - { - string currPkgNameVersion = $"{pkgToInstall.Name}{pkgVersion}"; - // Use HashSet lookup instead of Contains for O(1) performance - if (_packagesOnMachine.Contains(currPkgNameVersion)) - { - _cmdletPassedIn.WriteWarning($"Resource '{pkgToInstall.Name}' with version '{pkgVersion}' is already installed. If you would like to reinstall, please run the cmdlet again with the -Reinstall parameter"); - - // Remove from tracking list of packages to install. - _pkgNamesToInstall.RemoveAll(x => x.Equals(pkgToInstall.Name, StringComparison.InvariantCultureIgnoreCase)); + // Note: the already-installed check and ShouldProcess gate are handled by the caller on the pipeline thread. + return pkgToInstall; + } - return packagesHash; - } - } + /// + /// Downloads the parent package and its dependencies to the temporary path. Safe to run on worker threads: + /// all host messages are routed to the provided queues and drained by the caller on the pipeline thread. + /// + private ConcurrentDictionary DownloadParentAndDeps( + PSResourceInfo pkgToInstall, + string pkgVersion, + string tempInstallPath, + PSRepositoryInfo repository, + ServerApiCall currentServer, + ResponseUtil currentResponseUtil, + bool skipDependencyCheck, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs, + out bool success) + { + debugMsgs.Enqueue("In InstallHelper::DownloadParentAndDeps()"); + ConcurrentDictionary packagesHash = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); - if (packagesHash.ContainsKey(pkgToInstall.Name)) + List parentAndDeps = new List(); + if (!skipDependencyCheck) { - return packagesHash; + // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method. + parentAndDeps.AddRange(_findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs)); + debugMsgs.Enqueue("In InstallHelper::DownloadParentAndDeps(), found all dependencies"); } + parentAndDeps.Add(pkgToInstall); - ConcurrentDictionary updatedPackagesHash = packagesHash; - - // -WhatIf processing. - if (_savePkg && !_cmdletPassedIn.ShouldProcess($"Package to save: '{pkgToInstall.Name}', version: '{pkgVersion}'")) - { - updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) - { - { "isModule", "" }, - { "isScript", "" }, - { "psResourceInfoPkg", pkgToInstall }, - { "tempDirNameVersionPath", tempInstallPath }, - { "pkgVersion", "" }, - { "scriptPath", "" }, - { "installPath", "" } - }); - } - else if (!_cmdletPassedIn.ShouldProcess($"Package to install: '{pkgToInstall.Name}', version: '{pkgVersion}'")) - { - if (!updatedPackagesHash.ContainsKey(pkgToInstall.Name)) - { - updatedPackagesHash.TryAdd(pkgToInstall.Name, new Hashtable(StringComparer.InvariantCultureIgnoreCase) - { - { "isModule", "" }, - { "isScript", "" }, - { "psResourceInfoPkg", pkgToInstall }, - { "tempDirNameVersionPath", tempInstallPath }, - { "pkgVersion", "" }, - { "scriptPath", "" }, - { "installPath", "" } - }); - } - } - else - { - // Concurrent updates - // Find all dependencies - if (!skipDependencyCheck) - { - // concurrency updates - List parentAndDeps = _findHelper.FindDependencyPackages(currentServer, currentResponseUtil, pkgToInstall, repository).ToList(); - // List returned only includes dependencies, so we'll add the parent pkg to this list to pass on to installation method - parentAndDeps.Add(pkgToInstall); - - _cmdletPassedIn.WriteDebug("In InstallHelper::BeginPackageInstall(), found all dependencies"); - - return InstallParentAndDependencyPackages(parentAndDeps, currentServer, tempInstallPath, packagesHash, updatedPackagesHash, pkgToInstall); - } - else { - // If we don't install dependencies, we're only installing the parent pkg so we can short circut and simply install the parent pkg. - Stream responseStream = currentServer.InstallPackage(pkgToInstall.Name, pkgVersion, true, out ErrorRecord installNameErrRecord); - - if (installNameErrRecord != null) - { - errRecord = installNameErrRecord; - return packagesHash; - } - ConcurrentQueue errorMsgs = new ConcurrentQueue(); - ConcurrentQueue warningMsgs = new ConcurrentQueue(); - ConcurrentQueue debugMsgs = new ConcurrentQueue(); - ConcurrentQueue verboseMsgs = new ConcurrentQueue(); - - bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs) : - TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstall.Name, pkgVersion, pkgToInstall, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (!installedToTempPathSuccessfully) - { - return packagesHash; - } - } - } + ConcurrentDictionary updatedPackagesHash = InstallParentAndDependencyPackages( + parentAndDeps, currentServer, tempInstallPath, packagesHash, packagesHash, pkgToInstall, + errorMsgs, warningMsgs, debugMsgs, verboseMsgs); + success = errorMsgs.IsEmpty; return updatedPackagesHash; } - private ConcurrentDictionary InstallParentAndDependencyPackages(List parentAndDeps, ServerApiCall currentServer, string tempInstallPath, ConcurrentDictionary packagesHash, ConcurrentDictionary updatedPackagesHash, PSResourceInfo pkgToInstall) + private ConcurrentDictionary InstallParentAndDependencyPackages( + List parentAndDeps, + ServerApiCall currentServer, + string tempInstallPath, + ConcurrentDictionary packagesHash, + ConcurrentDictionary updatedPackagesHash, + PSResourceInfo pkgToInstall, + ConcurrentQueue errorMsgs, + ConcurrentQueue warningMsgs, + ConcurrentQueue debugMsgs, + ConcurrentQueue verboseMsgs) { - string warning = string.Empty; - ConcurrentQueue errorMsgs = new ConcurrentQueue(); - ConcurrentQueue verboseMsgs = new ConcurrentQueue(); - ConcurrentQueue debugMsgs = new ConcurrentQueue(); - ConcurrentQueue warningMsgs = new ConcurrentQueue(); - // TODO: figure out a good threshold and parallel count int processorCount = Environment.ProcessorCount; - _cmdletPassedIn.WriteDebug($"parentAndDeps.Count is {parentAndDeps.Count}, processor count is: {processorCount}"); + debugMsgs.Enqueue($"parentAndDeps.Count is {parentAndDeps.Count}, processor count is: {processorCount}"); if (parentAndDeps.Count > processorCount) { - _cmdletPassedIn.WriteDebug($"parentAndDeps.Count is greater than processor count"); + debugMsgs.Enqueue($"parentAndDeps.Count is greater than processor count"); // Set the maximum degree of parallelism to 32? (Invoke-Command has default of 32, that's where we got this number from) // If installing more than 3 packages, do so concurrently // If the number of dependencies is very small (e.g., ≤ CPU cores), parallelism may add overhead instead of improving speed. @@ -870,7 +885,7 @@ private ConcurrentDictionary InstallParentAndDependencyPackag // add async Stream responseStream = currentServer.InstallPackageAsync(depPkgName, depPkgVersion, true, errorMsgs, warningMsgs, debugMsgs, verboseMsgs).GetAwaiter().GetResult(); - if (errorMsgs.Count > 0) + if (!errorMsgs.IsEmpty) { verboseMsgs.Enqueue($"Error installing package '{depPkgName}'"); } @@ -891,8 +906,7 @@ private ConcurrentDictionary InstallParentAndDependencyPackag } }); - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (errorMsgs.Count > 0) + if (!errorMsgs.IsEmpty) { return packagesHash; } @@ -906,20 +920,17 @@ private ConcurrentDictionary InstallParentAndDependencyPackag { var pkgToInstallName = pkgToBeInstalled.Name; var pkgToInstallVersion = Utils.GetFullVersionString(pkgToBeInstalled.Version.ToString(), pkgToBeInstalled.Prerelease); - Stream responseStream = currentServer.InstallPackage(pkgToInstallName, pkgToInstallVersion, true, out ErrorRecord installNameErrRecord); + // Runs on worker threads when parent installs are parallelized; use the async overload to avoid cross-thread cmdlet stream writes. + Stream responseStream = currentServer.InstallPackageAsync(pkgToInstallName, pkgToInstallVersion, true, errorMsgs, warningMsgs, debugMsgs, verboseMsgs).GetAwaiter().GetResult(); - if (installNameErrRecord != null) + if (!errorMsgs.IsEmpty) { - _cmdletPassedIn.WriteError(installNameErrRecord); return packagesHash; } - //ErrorRecord tempSaveErrRecord = null, tempInstallErrRecord = null; bool installedToTempPathSuccessfully = _asNupkg ? TrySaveNupkgToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs) : TryInstallToTempPath(responseStream, tempInstallPath, pkgToInstallName, pkgToInstallVersion, pkgToBeInstalled, packagesHash, out updatedPackagesHash, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs); - if (!installedToTempPathSuccessfully) { return packagesHash; diff --git a/src/code/NuGetServerAPICalls.cs b/src/code/NuGetServerAPICalls.cs index d8c6b5ffe..d2b19c5d8 100644 --- a/src/code/NuGetServerAPICalls.cs +++ b/src/code/NuGetServerAPICalls.cs @@ -63,10 +63,9 @@ public override Task FindVersionAsync(string packageName, string ve }); var filterBuilder = queryBuilder.FilterBuilder; - // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. - filterBuilder.AddCriterion($"Id eq '{packageName}'"); - filterBuilder.AddCriterion($"NormalizedVersion eq '{packageName}'"); - +// We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. +filterBuilder.AddCriterion($"Id eq '{packageName}'"); +filterBuilder.AddCriterion($"NormalizedVersion eq '{version}'"); var requestUrl = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; string response = HttpRequestCallAsync(requestUrl, debugMsgs, out ErrorRecord errRecord); FindResults findResponse = new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); @@ -87,13 +86,46 @@ public override Task FindVersionAsync(string packageName, string ve public override Task FindVersionGlobbingAsync(string packageName, VersionRange versionRange, bool includePrerelease, ResourceType type, bool getOnlyLatest, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In NuGetServerAPICalls::FindVersionGlobbingAsync()"); - FindResults findResponse = FindVersionGlobbing(packageName, versionRange, includePrerelease, type, getOnlyLatest, out ErrorRecord errRecord); + List responses = new List(); + int skip = 0; + + var initialResponse = FindVersionGlobbingFromEndpointAsync(packageName, versionRange, includePrerelease, skip, getOnlyLatest, debugMsgs, out ErrorRecord errRecord); if (errRecord != null) { errorMsgs.Enqueue(errRecord); + return Task.FromResult(new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType)); } - return Task.FromResult(findResponse); + responses.Add(initialResponse); + + if (!getOnlyLatest) + { + int initialCount = GetCountFromResponse(initialResponse, out errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return Task.FromResult(new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType)); + } + + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; + + while (count > 0) + { + // skip 100 + skip += 100; + var tmpResponse = FindVersionGlobbingFromEndpointAsync(packageName, versionRange, includePrerelease, skip, getOnlyLatest, debugMsgs, out errRecord); + if (errRecord != null) + { + errorMsgs.Enqueue(errRecord); + return Task.FromResult(new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType)); + } + + responses.Add(tmpResponse); + count--; + } + } + + return Task.FromResult(new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType)); } /// /// Find method which allows for searching for all packages from a repository and returns latest version for each. @@ -122,7 +154,7 @@ public override FindResults FindAll(bool includePrerelease, ResourceType type, o return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType); } - int count = initialCount / 6000; + int count = (int)Math.Ceiling((double)initialCount / 6000) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -166,7 +198,7 @@ public override FindResults FindTags(string[] tags, bool includePrerelease, Reso return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType); } - int count = initialCount / 100; + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -238,7 +270,19 @@ public override FindResults FindName(string packageName, bool includePrerelease, public override Task FindNameAsync(string packageName, bool includePrerelease, ResourceType type, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In NuGetServerAPICalls::FindNameAsync()"); - FindResults findResponse = FindName(packageName, includePrerelease, type, out ErrorRecord errRecord); + var queryBuilder = new NuGetV2QueryBuilder(new Dictionary{ + { "id", $"'{packageName}'" }, + }); + var filterBuilder = queryBuilder.FilterBuilder; + + filterBuilder.AddCriterion(includePrerelease ? "IsAbsoluteLatestVersion" : "IsLatestVersion"); + + // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. + filterBuilder.AddCriterion($"Id eq '{packageName}'"); + + var requestUrl = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; + string response = HttpRequestCallAsync(requestUrl, debugMsgs, out ErrorRecord errRecord); + FindResults findResponse = new FindResults(stringResponse: new string[] { response }, hashtableResponse: emptyHashResponses, responseType: FindResponseType); if (errRecord != null) { errorMsgs.Enqueue(errRecord); @@ -310,7 +354,7 @@ public override FindResults FindNameGlobbing(string packageName, bool includePre return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType); } - int count = initialCount / 100; + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -355,7 +399,7 @@ public override FindResults FindNameGlobbingWithTag(string packageName, string[] return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType); } - int count = initialCount / 100; + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -404,7 +448,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange return new FindResults(stringResponse: responses.ToArray(), hashtableResponse: emptyHashResponses, responseType: FindResponseType); } - int count = initialCount / 100; + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; while (count > 0) { @@ -521,7 +565,19 @@ public override Stream InstallPackage(string packageName, string packageVersion, public override Task InstallPackageAsync(string packageName, string packageVersion, bool includePrerelease, ConcurrentQueue errorMsgs, ConcurrentQueue warningMsgs, ConcurrentQueue debugMsgs, ConcurrentQueue verboseMsgs) { debugMsgs.Enqueue("In NuGetServerAPICalls::InstallPackageAsync()"); - Stream results = InstallPackage(packageName, packageVersion, includePrerelease, out ErrorRecord errRecord); + Stream results = new MemoryStream(); + if (string.IsNullOrEmpty(packageVersion)) + { + errorMsgs.Enqueue(new ErrorRecord( + exception: new ArgumentNullException($"Package version could not be found for {packageName}"), + "PackageVersionNullOrEmptyError", + ErrorCategory.InvalidArgument, + _cmdletPassedIn)); + + return Task.FromResult(results); + } + + results = InstallVersionAsync(packageName, packageVersion, debugMsgs, out ErrorRecord errRecord); if (errRecord != null) { errorMsgs.Enqueue(errRecord); @@ -628,6 +684,55 @@ private HttpContent HttpRequestCallForContent(string requestUrl, out ErrorRecord return content; } + /// + /// Helper method that makes the HTTP request for install APIs on worker threads; enqueues diagnostics instead of writing to cmdlet streams. + /// + private HttpContent HttpRequestCallForContentAsync(string requestUrl, ConcurrentQueue debugMsgs, out ErrorRecord errRecord) + { + debugMsgs.Enqueue("In NuGetServerAPICalls::HttpRequestCallForContentAsync()"); + errRecord = null; + HttpContent content = null; + + try + { + debugMsgs.Enqueue($"Request url is: '{requestUrl}'"); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestUrl); + + content = SendRequestForContentAsync(request, _sessionClient).GetAwaiter().GetResult(); + } + catch (HttpRequestException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFailure", + ErrorCategory.ConnectionError , + this); + } + catch (ArgumentNullException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFailure", + ErrorCategory.InvalidData, + this); + } + catch (InvalidOperationException e) + { + errRecord = new ErrorRecord( + exception: e, + "HttpRequestFailure", + ErrorCategory.InvalidOperation, + this); + } + + if (string.IsNullOrEmpty(content?.ToString())) + { + debugMsgs.Enqueue("Response is empty"); + } + + return content; + } + /// /// Helper method that makes the HTTP request for the NuGet server protocol url passed in for async find APIs. /// This helper writes diagnostics to the provided debug queue and avoids cmdlet stream writes. @@ -912,6 +1017,25 @@ private string FindNameGlobbingWithTag(string packageName, string[] tags, bool i private string FindVersionGlobbing(string packageName, VersionRange versionRange, bool includePrerelease, int skip, bool getOnlyLatest, out ErrorRecord errRecord) { _cmdletPassedIn.WriteDebug("In NuGetServerAPICalls::FindVersionGlobbing()"); + var requestUrl = GetVersionGlobbingRequestUrl(packageName, versionRange, includePrerelease, skip, getOnlyLatest); + return HttpRequestCall(requestUrl, out errRecord); + } + + /// + /// Worker-thread counterpart of FindVersionGlobbing(); enqueues diagnostics instead of writing to cmdlet streams. + /// + private string FindVersionGlobbingFromEndpointAsync(string packageName, VersionRange versionRange, bool includePrerelease, int skip, bool getOnlyLatest, ConcurrentQueue debugMsgs, out ErrorRecord errRecord) + { + debugMsgs.Enqueue("In NuGetServerAPICalls::FindVersionGlobbingFromEndpointAsync()"); + var requestUrl = GetVersionGlobbingRequestUrl(packageName, versionRange, includePrerelease, skip, getOnlyLatest); + return HttpRequestCallAsync(requestUrl, debugMsgs, out errRecord); + } + + /// + /// Builds the FindPackagesById() request url for version-globbing searches. + /// + private string GetVersionGlobbingRequestUrl(string packageName, VersionRange versionRange, bool includePrerelease, int skip, bool getOnlyLatest) + { //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='blah'&includePrerelease=false&$filter= NormalizedVersion gt '1.0.0' and NormalizedVersion lt '2.2.5' and substringof('PSModule', Tags) eq true //https://www.powershellgallery.com/api/v2//FindPackagesById()?id='PowerShellGet'&includePrerelease=false&$filter= NormalizedVersion gt '1.1.1' and NormalizedVersion lt '2.2.5' // NormalizedVersion doesn't include trailing zeroes @@ -980,9 +1104,7 @@ private string FindVersionGlobbing(string packageName, VersionRange versionRange // We need to explicitly add 'Id eq ' whenever $filter is used, otherwise arbitrary results are returned. filterBuilder.AddCriterion($"Id eq '{packageName}'"); - var requestUrl = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; - - return HttpRequestCall(requestUrl, out errRecord); + return $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}"; } /// @@ -1040,6 +1162,29 @@ private Stream InstallVersion(string packageName, string version, out ErrorRecor return response.ReadAsStreamAsync().Result; } + /// + /// Worker-thread counterpart of InstallVersion(); enqueues diagnostics instead of writing to cmdlet streams. + /// + private Stream InstallVersionAsync(string packageName, string version, ConcurrentQueue debugMsgs, out ErrorRecord errRecord) + { + debugMsgs.Enqueue("In NuGetServerAPICalls::InstallVersionAsync()"); + var requestUrl = $"{Repository.Uri}/Packages(Id='{packageName}',Version='{version}')/Download"; + var response = HttpRequestCallForContentAsync(requestUrl, debugMsgs, out errRecord); + + if (response is null) + { + errRecord = new ErrorRecord( + new Exception($"No content was returned by repository '{Repository.Name}'"), + "InstallFailureContentNullNuGetServer", + ErrorCategory.InvalidResult, + this); + + return null; + } + + return response.ReadAsStreamAsync().Result; + } + /// /// Helper method that makes gets 'count' property from http response string. /// The count property is used to determine the number of total results found (for pagination). diff --git a/src/code/V2ServerAPICalls.cs b/src/code/V2ServerAPICalls.cs index 963ed5ed7..6ea3361d7 100644 --- a/src/code/V2ServerAPICalls.cs +++ b/src/code/V2ServerAPICalls.cs @@ -206,7 +206,7 @@ public override FindResults FindTags(string[] tags, bool includePrerelease, Reso if (initialScriptCount != 0) { responses.Add(initialScriptResponse); - int count = initialScriptCount / 100; + int count = (int)Math.Ceiling((double)initialScriptCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -242,7 +242,7 @@ public override FindResults FindTags(string[] tags, bool includePrerelease, Reso if (initialModuleCount != 0) { responses.Add(initialModuleResponse); - int count = initialModuleCount / 100; + int count = (int)Math.Ceiling((double)initialModuleCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -296,7 +296,7 @@ public override FindResults FindCommandOrDscResource(string[] tags, bool include if (initialCount != 0) { responses.Add(initialResponse); - int count = (int)Math.Ceiling((double)(initialCount / 100)); + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; while (count > 0) { @@ -596,7 +596,7 @@ public override FindResults FindNameGlobbing(string packageName, bool includePre return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } - int count = (int)Math.Ceiling((double)(initialCount / 100)); + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -648,7 +648,7 @@ public override FindResults FindNameGlobbingWithTag(string packageName, string[] return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType); } - int count = (int)Math.Ceiling((double)(initialCount / 100)); + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) { @@ -704,7 +704,7 @@ public override FindResults FindVersionGlobbing(string packageName, VersionRange if (!getOnlyLatest) { - int count = (int)Math.Ceiling((double)(initialCount / 100)); + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; while (count > 0) { @@ -1735,7 +1735,7 @@ public override async Task FindVersionGlobbingAsync(string packageN if (!getOnlyLatest) { - int count = (int)Math.Ceiling((double)(initialCount / 100)); + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; while (count > 0) { diff --git a/src/code/V3ServerAPICalls.cs b/src/code/V3ServerAPICalls.cs index 234fc513a..a66b35e9d 100644 --- a/src/code/V3ServerAPICalls.cs +++ b/src/code/V3ServerAPICalls.cs @@ -750,7 +750,7 @@ private FindResults FindVersionHelper(string packageName, string version, string return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v3FindResponseType); } - //_cmdletPassedIn.WriteDebug($"'{packageName}' version parsed as '{requiredVersion}'"); +debugMsgs.Enqueue($"'{packageName}' version parsed as '{requiredVersion}'"); string[] versionedResponses = GetVersionedPackageEntriesFromRegistrationsResource(packageName, catalogEntryProperty, isSearch: true, out errRecord, errorMsgs, debugMsgs, verboseMsgs); if (errRecord != null) @@ -1097,8 +1097,8 @@ private List GetVersionedPackageEntriesFromSearchQueryResource(stri // Get responses for all packages that contain the required tags pkgEntries.AddRange(GetJsonElementArr(query, dataName, out int initialCount, out errRecord, errorMsgs, debugMsgs, verboseMsgs).ToList()); - // check count (ie "totalHits") 425 ==> count/100 ~~> 4 calls ~~> + 1 = 5 calls - int count = initialCount / 100 + 1; + // check count (ie "totalHits") 425 ==> ceil(425/100) - 1 ~~> 4 more calls (initial page already fetched) + int count = (int)Math.Ceiling((double)initialCount / 100) - 1; // if more than 100 count, loop and add response to list while (count > 0) {