diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/DiskFiltersTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/DiskFiltersTests.cs index 90530448ff..4c87202ab8 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/DiskFiltersTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/DiskFiltersTests.cs @@ -341,6 +341,89 @@ public void DiskFiltersCanFilterOnSizeBiggerThanOnLinux() Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(3), result.ElementAt(1))); } + [Test] + [TestCase("SizeGreaterThan:3.7gb")] + [TestCase("SizeGreaterThan:3788.8mb")] + [TestCase("SizeGreaterThan:3879731.2kb")] + public void DiskFiltersCanFilterOnSizeBiggerThanUsingDecimalValues(string filterString) + { + // 3.7GB is 3972844748.8 bytes. The fraction should not get it rejected. + this.disks = this.mockFixture.CreateDisks(PlatformID.Unix, true); + this.disks.ElementAt(0).Properties["size"] = (long)5 * 1024 * 1024 * 1024; + this.disks.ElementAt(1).Properties["size"] = (long)3 * 1024 * 1024 * 1024; + this.disks.ElementAt(2).Properties["size"] = (long)2 * 1024 * 1024 * 1024; + this.disks.ElementAt(3).Properties["size"] = (long)7 * 1024 * 1024 * 1024; + + IEnumerable result = DiskFilters.FilterDisks(this.disks, filterString, PlatformID.Unix); + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(0), result.ElementAt(0))); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(3), result.ElementAt(1))); + } + + [Test] + public void DiskFiltersCanFilterOnSizeLessThanUsingDecimalValues() + { + this.disks = this.mockFixture.CreateDisks(PlatformID.Unix, true); + this.disks.ElementAt(0).Properties["size"] = (long)5 * 1024 * 1024 * 1024; + this.disks.ElementAt(1).Properties["size"] = (long)3 * 1024 * 1024 * 1024; + this.disks.ElementAt(2).Properties["size"] = (long)2 * 1024 * 1024 * 1024; + this.disks.ElementAt(3).Properties["size"] = (long)7 * 1024 * 1024 * 1024; + + string filterString = "SizeLessThan:3.7gb"; + IEnumerable result = DiskFilters.FilterDisks(this.disks, filterString, PlatformID.Unix); + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(1), result.ElementAt(0))); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(2), result.ElementAt(1))); + } + + [Test] + public void DiskFiltersCanFilterOnSizeEqualToUsingDecimalValues() + { + this.disks = this.mockFixture.CreateDisks(PlatformID.Unix, true); + this.disks.ElementAt(0).Properties["size"] = (long)5 * 1024 * 1024 * 1024; + this.disks.ElementAt(1).Properties["size"] = (long)3 * 1024 * 1024 * 1024; + this.disks.ElementAt(2).Properties["size"] = (long)2 * 1024 * 1024 * 1024; + this.disks.ElementAt(3).Properties["size"] = (long)7 * 1024 * 1024 * 1024; + + // 3.001GB is 3222299213.824 bytes; only the 3GB disk falls inside the 1% buffer. + string filterString = "SizeEqualTo:3.001gb"; + IEnumerable result = DiskFilters.FilterDisks(this.disks, filterString, PlatformID.Unix); + Assert.AreEqual(1, result.Count()); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(1), result.ElementAt(0))); + } + + [Test] + public void DiskFiltersCanFilterOnSizeGivenAsABareNumberOfBytes() + { + this.disks = this.mockFixture.CreateDisks(PlatformID.Unix, true); + this.disks.ElementAt(0).Properties["size"] = (long)5 * 1024 * 1024 * 1024; + this.disks.ElementAt(1).Properties["size"] = (long)3 * 1024 * 1024 * 1024; + this.disks.ElementAt(2).Properties["size"] = (long)2 * 1024 * 1024 * 1024; + this.disks.ElementAt(3).Properties["size"] = (long)7 * 1024 * 1024 * 1024; + + string filterString = "SizeGreaterThan:3972844748.8"; + IEnumerable result = DiskFilters.FilterDisks(this.disks, filterString, PlatformID.Unix); + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(0), result.ElementAt(0))); + Assert.IsTrue(object.ReferenceEquals(this.disks.ElementAt(3), result.ElementAt(1))); + } + + [Test] + [TestCase("SizeGreaterThan:abc")] + [TestCase("SizeLessThan:not-a-size")] + [TestCase("SizeEqualTo:gb")] + [TestCase("SizeGreaterThan:")] + public void DiskFiltersThrowOnAnInvalidSizeValue(string filterString) + { + this.disks = this.mockFixture.CreateDisks(PlatformID.Unix, true); + + EnvironmentSetupException error = Assert.Throws( + () => DiskFilters.FilterDisks(this.disks, filterString, PlatformID.Unix).ToList()); + + Assert.AreEqual(ErrorReason.DiskInformationNotAvailable, error.Reason); + Assert.IsTrue(error.Message.Contains("disk filter", StringComparison.OrdinalIgnoreCase)); + } + [Test] [Ignore("We can support this in the future if we add a 'Size' property to the Disk object for Windows (e.g. DiskPart -> list disks) and consider that in the filtering.")] public void DiskFiltersCanFilterOnSizeBiggerThanOnWindows() diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/Parser/TextParsingExtensionsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/Parser/TextParsingExtensionsTests.cs index c2fc0f3c12..b999b74a47 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/Parser/TextParsingExtensionsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/Parser/TextParsingExtensionsTests.cs @@ -6,6 +6,8 @@ namespace VirtualClient.Contracts.Parser using NUnit.Framework; using System; using System.Collections.Generic; + using System.Globalization; + using System.Threading; using VirtualClient.Common; using VirtualClient.TestExtensions; @@ -24,6 +26,87 @@ public void TextParsingExtensionsTranslateByteUnitAsExpected(string originalText Assert.IsTrue(string.Equals(TextParsingExtensions.TranslateByteUnit(originalText), expectedOutput)); } + [Test] + [TestCase("1.5kb", "1536")] + [TestCase("1.5mb", "1572864")] + [TestCase("1.5gb", "1610612736")] + [TestCase("0.5tb", "549755813888")] + [TestCase("2.5pb", "2814749767106560")] + [TestCase("100.75gb", "108179488768")] + [TestCase("2.5 gb", "2684354560")] + public void TextParsingExtensionsTranslateByteUnitSupportsDecimalValues(string originalText, string expectedOutput) + { + Assert.AreEqual(expectedOutput, TextParsingExtensions.TranslateByteUnit(originalText)); + } + + [Test] + [TestCase("3.7tb", "4068193022771.2")] + [TestCase("3.7gb", "3972844748.8")] + [TestCase("1.1kb", "1126.4")] + public void TextParsingExtensionsTranslateByteUnitSupportsFractionalByteCounts(string originalText, string expectedOutput) + { + // The units are powers of 1024, so something like 3.7GB does not land on a whole byte. Keep the + // fraction instead of rounding or throwing. + Assert.AreEqual(expectedOutput, TextParsingExtensions.TranslateByteUnit(originalText)); + } + + [Test] + [TestCase("8pb", 9007199254740992)] + [TestCase("64pb", 72057594037927936)] + [TestCase("8191pb", 9222246136947933184)] + public void TextParsingExtensionsTranslateByteUnitRemainsExactAcrossTheInt64Range(string originalText, long expectedBytes) + { + // double only holds whole numbers exactly up to 2^53 (~8PB). decimal covers the full Int64 range. + Assert.AreEqual(expectedBytes, TextParsingExtensions.TranslateByteUnitToBytes(originalText)); + Assert.AreEqual(expectedBytes.ToString(CultureInfo.InvariantCulture), TextParsingExtensions.TranslateByteUnit(originalText)); + } + + [Test] + [TestCase("en-US")] + [TestCase("de-DE")] + [TestCase("fr-FR")] + public void TextParsingExtensionsTranslateByteUnitIsNotAffectedByTheCurrentCulture(string culture) + { + CultureInfo originalCulture = Thread.CurrentThread.CurrentCulture; + + try + { + Thread.CurrentThread.CurrentCulture = new CultureInfo(culture); + + // Where ',' is the decimal separator, culture-sensitive parsing would read '1.5' as 15 or just fail. + Assert.AreEqual("1610612736", TextParsingExtensions.TranslateByteUnit("1.5gb")); + Assert.AreEqual("4068193022771.2", TextParsingExtensions.TranslateByteUnit("3.7tb")); + } + finally + { + Thread.CurrentThread.CurrentCulture = originalCulture; + } + } + + [Test] + [TestCase("1024", "1024")] + [TestCase("1.5", "1.5")] + [TestCase("100kb", "102400")] + [TestCase("3.7tb", "4068193022771.2")] + public void TextParsingExtensionsTryTranslateByteUnitAsExpected(string originalText, string expectedBytes) + { + Assert.IsTrue(TextParsingExtensions.TryTranslateByteUnit(originalText, out decimal bytes)); + Assert.AreEqual(decimal.Parse(expectedBytes, CultureInfo.InvariantCulture), bytes); + } + + [Test] + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + [TestCase("abc")] + [TestCase("gb")] + [TestCase("not-a-size")] + public void TextParsingExtensionsTryTranslateByteUnitHandlesInvalidValues(string originalText) + { + Assert.IsFalse(TextParsingExtensions.TryTranslateByteUnit(originalText, out decimal bytes)); + Assert.AreEqual(0m, bytes); + } + [Test] [TestCase("100kb", MetricUnit.Kilobytes, "100")] [TestCase("100mb", MetricUnit.Megabytes, "100")] diff --git a/src/VirtualClient/VirtualClient.Contracts/DiskExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/DiskExtensions.cs index ead774434d..864db3fbc3 100644 --- a/src/VirtualClient/VirtualClient.Contracts/DiskExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/DiskExtensions.cs @@ -133,7 +133,7 @@ public static long SizeInBytes(this DiskVolume volume, PlatformID platform) volume.Properties.TryGetValue(Disk.WindowsDiskProperties.Size, out IConvertible windowsSize); // Default to 0. windowsSize = (windowsSize == null) ? "0" : windowsSize; - result = Convert.ToInt64(TextParsingExtensions.TranslateByteUnit(windowsSize.ToString())); + result = (long)Math.Round(TextParsingExtensions.TranslateByteUnitToBytes(windowsSize.ToString()), MidpointRounding.AwayFromZero); } else if (platform == PlatformID.Unix) { @@ -159,7 +159,7 @@ public static long SizeInBytes(this Disk disk, PlatformID platform) { if (disk.Properties.TryGetValue(Disk.WindowsDiskProperties.Size, out IConvertible windowsSize)) { - result = Convert.ToInt64(TextParsingExtensions.TranslateByteUnit(windowsSize.ToString())); + result = (long)Math.Round(TextParsingExtensions.TranslateByteUnitToBytes(windowsSize.ToString()), MidpointRounding.AwayFromZero); } else if (disk.Volumes.Any()) { diff --git a/src/VirtualClient/VirtualClient.Contracts/DiskFilters.cs b/src/VirtualClient/VirtualClient.Contracts/DiskFilters.cs index c1440c3597..486d978f09 100644 --- a/src/VirtualClient/VirtualClient.Contracts/DiskFilters.cs +++ b/src/VirtualClient/VirtualClient.Contracts/DiskFilters.cs @@ -145,15 +145,15 @@ public static IEnumerable FilterDisks(IEnumerable disks, string filt break; case Filters.SizeGreaterThan: - disks = DiskFilters.SizeGreaterThanFilter(disks, platform, Convert.ToInt64(TextParsingExtensions.TranslateByteUnit(filterValue))); + disks = DiskFilters.SizeGreaterThanFilter(disks, platform, DiskFilters.ParseDiskSize(filterName, filterValue)); break; case Filters.SizeLessThan: - disks = DiskFilters.SizeLessThanFilter(disks, platform, Convert.ToInt64(TextParsingExtensions.TranslateByteUnit(filterValue))); + disks = DiskFilters.SizeLessThanFilter(disks, platform, DiskFilters.ParseDiskSize(filterName, filterValue)); break; case Filters.SizeEqualTo: - disks = DiskFilters.SizeEqualToFilter(disks, platform, Convert.ToInt64(TextParsingExtensions.TranslateByteUnit(filterValue))); + disks = DiskFilters.SizeEqualToFilter(disks, platform, DiskFilters.ParseDiskSize(filterName, filterValue)); break; case Filters.OsDisk: @@ -189,6 +189,20 @@ public static IEnumerable FilterDisks(IEnumerable disks, string filt return disks; } + private static decimal ParseDiskSize(string filterName, string filterValue) + { + // Sizes are kept as decimal because values like 3.7TB do not land on a whole number of bytes. + if (!TextParsingExtensions.TryTranslateByteUnit(filterValue, out decimal sizeInBytes) || sizeInBytes < 0) + { + throw new EnvironmentSetupException( + $"Invalid disk filter. The value '{filterValue}' supplied for the '{filterName}' disk filter is not a valid disk size. " + + $"Supply a size in bytes or a size with a unit (e.g. 1024, 100KB, 1.5GB, 3.7TB).", + ErrorReason.DiskInformationNotAvailable); + } + + return sizeInBytes; + } + private static IEnumerable BiggestSizeFilter(IEnumerable disks, PlatformID platform) { long biggestSize = disks.Max(d => d.SizeInBytes(platform)); @@ -204,21 +218,21 @@ private static IEnumerable SmallestSizeFilter(IEnumerable disks, Pla return disks; } - private static IEnumerable SizeGreaterThanFilter(IEnumerable disks, PlatformID platform, long size) + private static IEnumerable SizeGreaterThanFilter(IEnumerable disks, PlatformID platform, decimal size) { disks = disks.Where(d => d.SizeInBytes(platform) >= size); return disks; } - private static IEnumerable SizeEqualToFilter(IEnumerable disks, PlatformID platform, long size) + private static IEnumerable SizeEqualToFilter(IEnumerable disks, PlatformID platform, decimal size) { // Due to disks are not always sized exactly as defined, due to reserved partitions and disk headers, etc. // We are leaving a 1% buffer. - disks = disks.Where(d => d.SizeInBytes(platform) >= size * 0.99 && d.SizeInBytes(platform) <= size * 1.01); + disks = disks.Where(d => d.SizeInBytes(platform) >= size * 0.99m && d.SizeInBytes(platform) <= size * 1.01m); return disks; } - private static IEnumerable SizeLessThanFilter(IEnumerable disks, PlatformID platform, long size) + private static IEnumerable SizeLessThanFilter(IEnumerable disks, PlatformID platform, decimal size) { disks = disks.Where(d => d.SizeInBytes(platform) <= size); return disks; diff --git a/src/VirtualClient/VirtualClient.Contracts/Parser/TextParsingExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Parser/TextParsingExtensions.cs index d68419f0cb..cfeb7204a3 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Parser/TextParsingExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Parser/TextParsingExtensions.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Contracts using System; using System.Collections.Generic; using System.Data; + using System.Globalization; using System.Linq; using System.Text.RegularExpressions; using YamlDotNet.Core.Tokens; @@ -300,6 +301,10 @@ public static string TranslateNumericUnit(string text) /// /// Translate the unit of number in a text. Example: 1K->1024 and 1M->1,048,576. /// + /// + /// Uses decimal rather than double so that fractional sizes (e.g. 3.7TB) work and byte counts stay + /// exact across the whole Int64 range. Double starts losing whole numbers above 2^53 (~8PB). + /// /// Original text. public static string TranslateByteUnit(string text) { @@ -308,7 +313,7 @@ public static string TranslateByteUnit(string text) Match thousandMatch = Regex.Match(text, thousandRegex.ToString(), thousandRegex.Options); if (thousandMatch.Success) { - text = Convert.ToString(Convert.ToDouble(thousandMatch.Groups[1].Value) * 1024); + text = TextParsingExtensions.ToByteUnitString(TextParsingExtensions.ParseByteUnitValue(thousandMatch.Groups[1].Value) * 1024m); } // Unit million: M, m, MB, mb, MiB @@ -316,7 +321,7 @@ public static string TranslateByteUnit(string text) Match millionMatch = Regex.Match(text, millionRegex.ToString(), millionRegex.Options); if (millionMatch.Success) { - text = Convert.ToString(Convert.ToDouble(millionMatch.Groups[1].Value) * 1024 * 1024); + text = TextParsingExtensions.ToByteUnitString(TextParsingExtensions.ParseByteUnitValue(millionMatch.Groups[1].Value) * 1024m * 1024m); } // Unit giga: G, g, GB, gb, GiB @@ -324,7 +329,7 @@ public static string TranslateByteUnit(string text) Match gigaMatch = Regex.Match(text, gigaRegex.ToString(), gigaRegex.Options); if (gigaMatch.Success) { - text = Convert.ToString(Convert.ToDouble(gigaMatch.Groups[1].Value) * 1024 * 1024 * 1024); + text = TextParsingExtensions.ToByteUnitString(TextParsingExtensions.ParseByteUnitValue(gigaMatch.Groups[1].Value) * 1024m * 1024m * 1024m); } // Unit tera: T, t, TB, tb, TiB @@ -332,7 +337,7 @@ public static string TranslateByteUnit(string text) Match teraMatch = Regex.Match(text, teraRegex.ToString(), teraRegex.Options); if (teraMatch.Success) { - text = Convert.ToString(Convert.ToDouble(teraMatch.Groups[1].Value) * 1024 * 1024 * 1024 * 1024); + text = TextParsingExtensions.ToByteUnitString(TextParsingExtensions.ParseByteUnitValue(teraMatch.Groups[1].Value) * 1024m * 1024m * 1024m * 1024m); } // Unit peta: P, p, PB, pb, PiB @@ -340,12 +345,50 @@ public static string TranslateByteUnit(string text) Match petaMatch = Regex.Match(text, petaRegex.ToString(), petaRegex.Options); if (petaMatch.Success) { - text = Convert.ToString(Convert.ToDouble(petaMatch.Groups[1].Value) * 1024 * 1024 * 1024 * 1024 * 1024); + text = TextParsingExtensions.ToByteUnitString(TextParsingExtensions.ParseByteUnitValue(petaMatch.Groups[1].Value) * 1024m * 1024m * 1024m * 1024m * 1024m); } return text; } + /// + /// Translate the unit of number in a text to a byte count. Example: 1KB->1024 and 3.7TB->4068193022771.2. + /// + /// Original text. + public static decimal TranslateByteUnitToBytes(string text) + { + return decimal.Parse(TextParsingExtensions.TranslateByteUnit(text), NumberStyles.Float, CultureInfo.InvariantCulture); + } + + /// + /// Translate the unit of number in a text to a byte count. Returns false if the text is not a valid size. + /// + /// Original text. + /// The number of bytes the text represents. + public static bool TryTranslateByteUnit(string text, out decimal bytes) + { + bytes = 0; + + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + try + { + bytes = TextParsingExtensions.TranslateByteUnitToBytes(text); + return true; + } + catch (FormatException) + { + return false; + } + catch (OverflowException) + { + return false; + } + } + /// /// Translate storage by unit provided. /// @@ -353,7 +396,7 @@ public static string TranslateByteUnit(string text) /// storage unit for eg bytes,kilobytes,megabytes,etc. public static string TranslateStorageByUnit(string text, string metricUnit) { - double byteUnitStorage = Convert.ToDouble(TranslateByteUnit(text)); + double byteUnitStorage = Convert.ToDouble(TranslateByteUnit(text), CultureInfo.InvariantCulture); string result = null; switch (metricUnit) @@ -461,5 +504,17 @@ public static IList ParseMetricsByRegexCaptureGroups( return result; } + + private static decimal ParseByteUnitValue(string value) + { + return decimal.Parse(value, NumberStyles.Float, CultureInfo.InvariantCulture); + } + + private static string ToByteUnitString(decimal value) + { + // No trailing zeros and no scientific notation, so whole byte counts still come out as plain + // integers for the callers that run long.TryParse over the result. + return value.ToString("0.############################", CultureInfo.InvariantCulture); + } } }