diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/MongoDB/MongoDBServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/MongoDB/MongoDBServerExecutorTests.cs index e188862b90..08446fde78 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/MongoDB/MongoDBServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/MongoDB/MongoDBServerExecutorTests.cs @@ -342,6 +342,77 @@ public async Task MongoDBServerExecutor_InitializeAsync_WithDiskFilter_CallsInit "ServerApiClient should be initialized"); } + [Test] + public async Task MongoDBServerExecutor_ConfigureDisk_ResolvesTheMongoDBServiceUserForThePlatform() + { + // SETUP: A DiskFilter triggers the disk configuration workflow. + this.mockFixture.Parameters["DiskFilter"] = "BiggestSize"; + this.mockFixture.Parameters["DiskDevicePath"] = "/dev/nvme0n1"; + + var volumes = new List(); + var disk = new Disk(index: 0, devicePath: "/dev/nvme0n1", volumes: volumes, properties: null); + this.mockFixture.DiskManager.Setup(dm => dm.GetDisksAsync(It.IsAny())) + .ReturnsAsync(new List { disk }); + + List commandsExecuted = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, args, workingDir) => + { + commandsExecuted.Add($"{exe} {args}"); + this.mockFixture.Process.StandardOutput.Clear(); + this.mockFixture.Process.StandardOutput.Append("{ \"ok\" : 1 }"); + this.mockFixture.Process.ExitCode = 0; + return this.mockFixture.Process; + }; + + var executor = new TestableMongoDBServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + // ACT + await executor.InitializeAsync(EventContext.Persisted(), CancellationToken.None); + + // ASSERT: The MongoDB service account is named 'mongodb' by Debian/Ubuntu packages but + // 'mongod' by RPM packages (Azure Linux, RHEL, Fedora). Hardcoding either one leaves the + // data directory owned by root on the other, and mongod then fails to create its journal. + string chownCommand = commandsExecuted.FirstOrDefault(cmd => cmd.Contains("chown", StringComparison.OrdinalIgnoreCase)); + + Assert.IsNotNull(chownCommand, "The data directory ownership command should have been executed."); + + Assert.IsFalse( + chownCommand.Contains("chown -R mongodb:mongodb", StringComparison.OrdinalIgnoreCase), + "The ownership command must not hardcode the Debian-only 'mongodb' account."); + + Assert.IsTrue( + chownCommand.Contains("id -u mongod", StringComparison.OrdinalIgnoreCase), + "The ownership command should probe for the RPM 'mongod' account."); + + Assert.IsTrue( + chownCommand.Contains("echo mongodb", StringComparison.OrdinalIgnoreCase), + "The ownership command should fall back to the Debian 'mongodb' account."); + } + + [Test] + public async Task MongoDBServerExecutor_InitializeAsync_OpensTheMongoDBPortOnTheLocalFirewall() + { + // SETUP: Capture the firewall entries the executor asks to be opened. + List firewallEntries = new List(); + this.mockFixture.FirewallManager + .Setup(mgr => mgr.EnableInboundConnectionsAsync(It.IsAny>(), It.IsAny())) + .Callback, CancellationToken>((entries, token) => firewallEntries.AddRange(entries)) + .Returns(Task.CompletedTask); + + this.mockFixture.Parameters["Port"] = 27017; + + var executor = new TestableMongoDBServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + // ACT + await executor.InitializeAsync(EventContext.Persisted(), CancellationToken.None); + + // ASSERT: Distros such as Azure Linux apply a default-deny inbound policy. Without opening + // the port, the YCSB client times out connecting to the server and loads zero records. + Assert.AreEqual(1, firewallEntries.Count, "The MongoDB port should have been opened on the local firewall."); + Assert.AreEqual("tcp", firewallEntries[0].Protocol); + CollectionAssert.AreEqual(new List { 27017 }, firewallEntries[0].Ports.ToList()); + } + [Test] public async Task MongoDBServerExecutor_InitializeAsync_CallsConfigureBindAddressAndStartServer() { diff --git a/src/VirtualClient/VirtualClient.Actions/MongoDB/MongoDBServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/MongoDB/MongoDBServerExecutor.cs index 2d52e0e828..a4d6544b0d 100644 --- a/src/VirtualClient/VirtualClient.Actions/MongoDB/MongoDBServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/MongoDB/MongoDBServerExecutor.cs @@ -24,6 +24,16 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64")] public class MongoDBServerExecutor : MongoDBExecutor { + /// + /// The MongoDB service account created by RPM-based packages (Azure Linux, RHEL, Fedora). + /// + private const string RpmServiceUser = "mongod"; + + /// + /// The MongoDB service account created by Debian-based packages (Ubuntu, Debian). + /// + private const string DebianServiceUser = "mongodb"; + private IFileSystem fileSystem; private ISystemManagement systemManagement; private bool disposed; @@ -68,6 +78,9 @@ protected override async Task InitializeAsync(EventContext telemetryContext, Can this.InitializeApiClients(); + await MongoDBServerExecutor.OpenFirewallPortsAsync(this.Port, this.systemManagement.FirewallManager, cancellationToken) + .ConfigureAwait(false); + // Initialize disk if DiskFilter is specified if (!string.IsNullOrWhiteSpace(this.DiskFilter)) { @@ -122,6 +135,25 @@ protected override void Dispose(bool disposing) } } + /// + /// Opens the MongoDB port on the local firewall so that the client instance is able to + /// connect to the MongoDB server. Distros such as Azure Linux apply a default-deny policy + /// to inbound traffic, so the port must be opened explicitly. + /// + private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallManager, CancellationToken cancellationToken) + { + return firewallManager.EnableInboundConnectionsAsync( + new List + { + new FirewallEntry( + "MongoDB: Allow Multiple Machines communications", + "Allows individual machine instances to communicate with other machine in client-server scenario", + "tcp", + new List { port }) + }, + cancellationToken); + } + /// /// Configures MongoDB to bind to all network interfaces. /// @@ -266,10 +298,16 @@ await this.ExecuteMongoDBCommandAsync( telemetryContext, cancellationToken).ConfigureAwait(false); - // Set permissions + // Set permissions. The MongoDB service account name differs by package format: + // Debian/Ubuntu packages create 'mongodb' whereas RPM-based distributions + // (Azure Linux, RHEL, Fedora) create 'mongod'. Resolve it at runtime so the + // data directory is owned by the account mongod actually runs as. + string resolveServiceUser = $"MONGO_USER=$(id -u {MongoDBServerExecutor.RpmServiceUser} >/dev/null 2>&1 && echo {MongoDBServerExecutor.RpmServiceUser} || echo {MongoDBServerExecutor.DebianServiceUser}); " + + $"sudo chown -R $MONGO_USER:$MONGO_USER {mongoDataPath}"; + await this.ExecuteMongoDBCommandAsync( "bash", - $"-c \"sudo chown -R mongodb:mongodb {mongoDataPath}\"", + $"-c \"{resolveServiceUser}\"", "SetPermissions", telemetryContext, cancellationToken).ConfigureAwait(false); diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentTests.cs index 561f0ddfb5..2ff93c702e 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentTests.cs @@ -724,6 +724,121 @@ public void VirtualClientComponentIsSupportedRespectsSupportedPlatformAttribute( Assert.IsFalse(VirtualClientComponent.IsSupported(component)); } + [Test] + public void VirtualClientComponentSupportedLinuxDistributionsIsEmptyWhenTheParameterIsNotDefined() + { + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsNotNull(component.SupportedLinuxDistributions); + Assert.IsEmpty(component.SupportedLinuxDistributions); + } + + [Test] + public void VirtualClientComponentSupportedLinuxDistributionsParsesDelimitedValues() + { + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Debian,Ubuntu"; + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + CollectionAssert.AreEqual(new List { "Debian", "Ubuntu" }, component.SupportedLinuxDistributions); + } + + [Test] + public void VirtualClientComponentIsSupportedWhenTheSupportedLinuxDistributionsParameterIsNotDefined() + { + // A component that does not define the parameter is not filtered on the Linux distribution + // at all and thus executes on any distribution. + this.mockFixture.Setup(PlatformID.Unix); + this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora); + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsTrue(component.IsSupported()); + } + + [Test] + [TestCase("AzureLinux")] + [TestCase("Debian,Ubuntu,AzureLinux")] + [TestCase("azurelinux")] + [TestCase("AZURELINUX")] + [TestCase(" AzureLinux , Ubuntu ")] + public void VirtualClientComponentIsSupportedWhenTheLinuxDistributionMatchesTheSupportedLinuxDistributions(string supportedDistributions) + { + this.mockFixture.Setup(PlatformID.Unix); + this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora); + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = supportedDistributions; + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsTrue(component.IsSupported()); + } + + [Test] + [TestCase("Debian,Ubuntu")] + [TestCase("Ubuntu")] + [TestCase("Fedora")] + public void VirtualClientComponentIsNotSupportedWhenTheLinuxDistributionDoesNotMatchTheSupportedLinuxDistributions(string supportedDistributions) + { + this.mockFixture.Setup(PlatformID.Unix); + this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora); + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = supportedDistributions; + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsFalse(component.IsSupported()); + } + + [Test] + public void VirtualClientComponentIsSupportedWhenTheLinuxDistributionMatchesADownstreamDistribution() + { + // Ubuntu is a downstream distribution of Debian. The match is made on the distribution + // itself and not on the upstream distribution. + this.mockFixture.Setup(PlatformID.Unix); + this.SetupLinuxDistribution(LinuxDistribution.Ubuntu, LinuxUpstreamDistribution.Debian); + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Debian,Ubuntu"; + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsTrue(component.IsSupported()); + } + + [Test] + public void VirtualClientComponentIsNotSupportedOnNonLinuxSystemsWhenTheSupportedLinuxDistributionsParameterIsDefined() + { + // A component defining the parameter is describing Linux-specific behavior and thus + // is never supported on non-Linux systems. + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Ubuntu"; + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsFalse(component.IsSupported()); + } + + [Test] + public void VirtualClientComponentIsNotSupportedWhenTheSupportedPlatformsDoNotMatchEvenIfTheLinuxDistributionMatches() + { + this.mockFixture.Setup(PlatformID.Unix, System.Runtime.InteropServices.Architecture.X64); + this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora); + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedPlatforms)] = "linux-arm64"; + this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "AzureLinux"; + + TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.IsFalse(component.IsSupported()); + } + + private void SetupLinuxDistribution(LinuxDistribution distribution, LinuxUpstreamDistribution upstreamDistribution) + { + this.mockFixture.SystemManagement + .Setup(sm => sm.GetLinuxDistributionAsync(It.IsAny())) + .ReturnsAsync(new LinuxDistributionInfo + { + Name = distribution.ToString(), + Distribution = distribution, + UpstreamDistribution = upstreamDistribution + }); + } + private class TestVirtualClientComponent : VirtualClientComponent { public TestVirtualClientComponent(VirtualClientComponent component) @@ -743,6 +858,11 @@ public TestVirtualClientComponent(IServiceCollection dependencies, IDictionary public DateTime StartTime { get; private set; } + /// + /// Parameter describes the Linux distributions (e.g. AzureLinux, Ubuntu) for which the component + /// is supported. A component defining this parameter is never executed on non-Linux systems. + /// + public IEnumerable SupportedLinuxDistributions + { + get + { + this.Parameters.TryGetCollection(nameof(this.SupportedLinuxDistributions), out IEnumerable distributions); + return distributions ?? Array.Empty(); + } + } + /// /// Parameter describes the platform/architectures for which the component is supported. /// @@ -943,6 +956,10 @@ protected virtual bool IsSupported() { isSupported = false; } + else if (this.SupportedLinuxDistributions?.Any() == true && !this.IsSupportedLinuxDistribution()) + { + isSupported = false; + } else if (this.Layout?.Clients?.Count() >= 2 && this.Roles?.Any() == true) { // Execution Criteria @@ -1018,5 +1035,21 @@ private bool IsMe(ClientInstance clientInstance) return isMatch; } + + private bool IsSupportedLinuxDistribution() + { + bool isSupported = false; + if (this.Platform == PlatformID.Unix) + { + LinuxDistributionInfo distribution = this.systemInfo.GetLinuxDistributionAsync(CancellationToken.None) + .GetAwaiter().GetResult(); + + isSupported = this.SupportedLinuxDistributions.Contains( + distribution.Distribution.ToString(), + StringComparer.OrdinalIgnoreCase); + } + + return isSupported; + } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MONGODB-YCSB.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MONGODB-YCSB.json index a70ac9ef05..54208f7ba5 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MONGODB-YCSB.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MONGODB-YCSB.json @@ -3,7 +3,7 @@ "Metadata": { "RecommendedMinimumExecutionTime": "00:15:00", "SupportedPlatforms": "linux-x64,linux-arm64", - "SupportedOperatingSystems": "Ubuntu", + "SupportedOperatingSystems": "AzureLinux,Ubuntu", "Notes": "Database sizes vary based on RecordCount: Small (500000 records) ~8-10 GB, Medium (2500000 records) ~40-50 GB, Large (20000000 records) ~320-400 GB, XLarge (55000000 records) ~900 GB-1 TB. Default configuration uses Medium size (~40-50 GB) with 2500000 records." }, "Parameters": { @@ -107,7 +107,7 @@ "MetricScenario": "Read_Latest_95_Read_5_Insert_{RecordCount}_Records", "Database": "$.Parameters.Database", "Port": "$.Parameters.Port", - "RunCommand": "run {Database} -s -p maxexecutiontime={Duration.TotalSeconds} -p operationcount={OperationCount} -p recordcount={RecordCount} -threads {ThreadCount} -p fieldcount={FieldCount} -p fieldlength={FieldLength} -p mongodb.url=mongodb://{ServerIP}:{Port}/ycsb -P {PackagePath:ycsb}/ycsb-0.17.0/workloads/workloadd", + "RunCommand": "run {Database} -s -p maxexecutiontime={Duration.TotalSeconds} -p operationcount={OperationCount} -p recordcount={RecordCount} -threads {ThreadCount} -p fieldcount={FieldCount} -p fieldlength={FieldLength} -p mongodb.url=mongodb://{ServerIP}:{Port}/ycsb -p mongodb.upsert=true -P {PackagePath:ycsb}/ycsb-0.17.0/workloads/workloadd", "Duration": "$.Parameters.Duration", "ThreadCount": "$.Parameters.ThreadCount", "OperationCount": "5000000", @@ -126,7 +126,7 @@ "MetricScenario": "Short_Range_Scan_95_Scan_5_Insert_{RecordCount}_Records", "Database": "$.Parameters.Database", "Port": "$.Parameters.Port", - "RunCommand": "run {Database} -s -p maxexecutiontime={Duration.TotalSeconds} -p operationcount={OperationCount} -p recordcount={RecordCount} -threads {ThreadCount} -p fieldcount={FieldCount} -p fieldlength={FieldLength} -p mongodb.url=mongodb://{ServerIP}:{Port}/ycsb -P {PackagePath:ycsb}/ycsb-0.17.0/workloads/workloade", + "RunCommand": "run {Database} -s -p maxexecutiontime={Duration.TotalSeconds} -p operationcount={OperationCount} -p recordcount={RecordCount} -threads {ThreadCount} -p fieldcount={FieldCount} -p fieldlength={FieldLength} -p mongodb.url=mongodb://{ServerIP}:{Port}/ycsb -p mongodb.upsert=true -P {PackagePath:ycsb}/ycsb-0.17.0/workloads/workloade", "Duration": "$.Parameters.Duration", "ThreadCount": "$.Parameters.ThreadCount", "OperationCount": "5000000", @@ -173,10 +173,10 @@ "Type": "LinuxPackageInstallation", "Parameters": { "Scenario": "InstallMongoPrereqs", - "Packages-Apt": "gnupg,curl", - "Packages-Dnf": "gnupg,curl", - "Packages-Yum": "gnupg,curl", - "Packages-Zypper": "gpg2,curl" + "Packages-Apt": "gnupg,curl,lshw", + "Packages-Dnf": "gnupg2,curl,lshw", + "Packages-Yum": "gnupg2,curl,lshw", + "Packages-Zypper": "gpg2,curl,lshw" } }, { @@ -184,6 +184,7 @@ "Parameters": { "Scenario": "MongoGPGKey", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "curl -fsSL https://www.mongodb.org/static/pgp/server-8.0.asc -o server-8.0.asc" } }, @@ -192,6 +193,7 @@ "Parameters": { "Scenario": "MongoGPGDearmor", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/mongodb-server-8.0.gpg server-8.0.asc" } }, @@ -200,6 +202,7 @@ "Parameters": { "Scenario": "MongoAddRepository", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "sudo sh -c \"echo 'deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] https://repo.mongodb.org/apt/ubuntu noble/mongodb-org/8.0 multiverse' > /etc/apt/sources.list.d/mongodb-org-8.0.list\"" } }, @@ -208,14 +211,43 @@ "Parameters": { "Scenario": "MongoUpdatePackageList", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "sudo apt-get update" } }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "MongoImportGPGKey", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "AzureLinux", + "Command": "sudo rpm --import https://pgp.mongodb.com/server-8.0.asc" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "MongoCreateDnfRepository", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "AzureLinux", + "Command": "sudo sh -c \"echo '[mongodb-org-8.0]\nname=MongoDB Repository\nbaseurl=https://repo.mongodb.org/yum/redhat/9/mongodb-org/8.0/$basearch/\ngpgcheck=1\nenabled=1\ngpgkey=https://pgp.mongodb.com/server-8.0.asc' > /etc/yum.repos.d/mongodb-org-8.0.repo\"" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "MongoRefreshPackageMetadata", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "AzureLinux", + "Command": "sudo dnf makecache --refresh" + } + }, { "Type": "ExecuteCommand", "Parameters": { "Scenario": "MongoInstallServer", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "sudo apt-get install -y mongodb-org", "Role": "Server" } @@ -225,10 +257,31 @@ "Parameters": { "Scenario": "MongoInstallClient", "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "Debian,Ubuntu", "Command": "sudo apt-get install -y mongodb-mongosh", "Role": "Client" } }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "MongoInstallServer", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "AzureLinux", + "Command": "sudo dnf install -y mongodb-org", + "Role": "Server" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "MongoInstallClient", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedLinuxDistributions": "AzureLinux", + "Command": "sudo dnf install -y mongodb-mongosh", + "Role": "Client" + } + }, { "Type": "ExecuteCommand", "Parameters": { diff --git a/website/docs/workloads/mongodb/mongodb-profiles.md b/website/docs/workloads/mongodb/mongodb-profiles.md index f5cad62fab..61acb18c3a 100644 --- a/website/docs/workloads/mongodb/mongodb-profiles.md +++ b/website/docs/workloads/mongodb/mongodb-profiles.md @@ -50,6 +50,7 @@ This profile loads a dataset into MongoDB and then runs various read, write, sca * linux-arm64 * **Supported Operating Systems** + * AzureLinux * Ubuntu * **Supports Disconnected Scenarios**