diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..474c817959b6 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,53 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + +4.22.1.0 > 4.23.0.0: + * VPC tiers created without an explicit network ACL now default to the + default_deny ACL (matching private gateway behavior) instead of being + persisted without any ACL, when the network offering supports the + NetworkACL service. Automation that created tiers without the aclid + parameter and relied on the previous implicit egress-allow behavior of + VR-based tiers should pass an explicit aclid. + + * Kubernetes clusters can no longer be created on VPC tiers that have no + network ACL attached or that use the immutable default_deny ACL; the API + now fails fast at validation instead of failing partway through cluster + provisioning. Attach an ACL that allows the required traffic (for example + default_allow or a custom ACL) to the tier before creating the cluster. + + * createVpnGateway now validates up front that the VPC's offering provides + the Vpn service through an available Site-to-Site VPN provider, returning + a clear error instead of "Cannot found source nat ip". + + * NSX: Site-to-Site VPN is now implemented natively for NSX NAT-mode VPCs. + The Vpn service of the seeded "VPC offering with NSX - NAT Mode" is backed + by route-based IPsec sessions terminating on the VPC's Tier-1 gateway. A + dedicated public IP is acquired for the VPN local endpoint when the VPN + gateway is created (or the IP passed via the ipaddressid parameter is + used) and released when the gateway is deleted. Remote peers must use + routed (VTI-style) IPsec configuration; policy-based peers requiring + strict subnet selectors are not supported. Some cryptographic options + permitted by CloudStack are rejected by NSX (3des, md5, aes192, + modp6144 and above, IKE lifetime below 21600 seconds) with a clear error. + + * NSX network offerings can now reference existing IP discovery, MAC + discovery, and segment security profiles. New NSX segments created from + those offerings bind the selected profiles in the same hierarchical + request as segment creation. Existing networks cannot be moved to an + offering with different profile bindings; operators must create + replacement networks when profile changes are required. + + * CloudStack-managed Kubernetes clusters now record explicit ownership of + their generated firewall, port-forwarding, load-balancing, and VPC ACL + rules. A reconcile API restores missing or drifted managed rules without + adopting unrelated customer policy. Existing clusters remain + legacy-unmanaged; a root administrator must validate and adopt their exact + rule topology before non-direct clusters can use managed start, topology, + or reconciliation workflows. + + * NSX zones can register operator-staged Tier-0 or VRF gateways and assign + them to accounts or domains through dedicated administrative APIs. The + feature is disabled by default (`nsx.vrf.scope=NONE`). CloudStack claims + and releases registered gateway records but does not create or delete the + physical NSX gateways, uplinks, or BGP configuration. diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index c32bb711c0f2..eee2540ae3ec 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -89,6 +89,10 @@ public interface NetworkService { IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Boolean displayIp, String ipaddress) throws ResourceAllocationException, InsufficientAddressCapacityException, ConcurrentOperationException; + IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Long vpcId, Boolean displayIp, + String ipaddress) throws ResourceAllocationException, InsufficientAddressCapacityException, + ConcurrentOperationException; + IpAddress reserveIpAddress(Account account, Boolean displayIp, Long ipAddressId) throws ResourceAllocationException; IpAddress reserveIpAddressWithVlanDetail(Account account, DataCenter zone, Boolean displayIp, String vlanDetailKey) throws ResourceAllocationException; diff --git a/api/src/main/java/com/cloud/network/element/Site2SiteVpnServiceProvider.java b/api/src/main/java/com/cloud/network/element/Site2SiteVpnServiceProvider.java index dd451324a72e..40829a5c5a7a 100644 --- a/api/src/main/java/com/cloud/network/element/Site2SiteVpnServiceProvider.java +++ b/api/src/main/java/com/cloud/network/element/Site2SiteVpnServiceProvider.java @@ -17,11 +17,55 @@ package com.cloud.network.element; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.IpAddress; +import com.cloud.network.Site2SiteCustomerGateway; import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; +import com.cloud.network.vpc.Vpc; import com.cloud.utils.component.Adapter; public interface Site2SiteVpnServiceProvider extends Adapter { + default void validateSite2SiteVpnCustomerGateway(Site2SiteCustomerGateway customerGateway) { + } + boolean startSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException; boolean stopSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException; + + /** + * Permanently removes a provider-side connection. This is distinct from stop: providers + * may disable a tunnel while retaining its profiles for an immediate reconnect, but deletion + * must remove all objects owned by the CloudStack connection. + */ + default boolean deleteSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException { + return stopSite2SiteVpn(conn); + } + + /** + * Lets the provider supply the public IP the VPN gateway should terminate on, instead of the + * VPC source NAT IP. Providers that terminate VPN on an external gateway (e.g. NSX Tier-1) + * acquire and return a dedicated IP here; requestedIp, when not null, is the IP the caller + * asked for and must be validated by the provider. Returning null means the provider has no + * preference and the manager falls back to the default IP selection. + */ + default IpAddress acquireVpnGatewayIp(Vpc vpc, IpAddress requestedIp) { + return null; + } + + /** + * Counterpart of {@link #acquireVpnGatewayIp(Vpc, IpAddress)}: invoked when a VPN gateway is + * deleted so the provider can tear down external VPN resources and release the gateway IP if + * it was acquired by the provider. + */ + default void releaseVpnGatewayIp(Site2SiteVpnGateway gateway) { + } + + /** + * Identifies a gateway previously owned by this provider. This is used during teardown when + * an offering has been edited since the gateway was created and the current service map no + * longer advertises the provider. + */ + default boolean ownsVpnGateway(Site2SiteVpnGateway gateway) { + return false; + } } diff --git a/api/src/main/java/com/cloud/network/nsx/NsxService.java b/api/src/main/java/com/cloud/network/nsx/NsxService.java index 1adb7461cc09..83f76346115f 100644 --- a/api/src/main/java/com/cloud/network/nsx/NsxService.java +++ b/api/src/main/java/com/cloud/network/nsx/NsxService.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.network.nsx; +import java.util.List; + import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.network.IpAddress; @@ -23,6 +25,10 @@ public interface NsxService { + static String getVrfZoneLockName(long zoneId) { + return String.format("NsxVrfGateway.Zone.%s", zoneId); + } + ConfigKey NSX_API_FAILURE_RETRIES = new ConfigKey<>("Advanced", Integer.class, "nsx.api.failure.retries", "30", "Number of retries for NSX API operations in case of failures", @@ -32,7 +38,22 @@ public interface NsxService { "Waiting time (in seconds) before retrying an NSX API operation in case of failure", true, ConfigKey.Scope.Zone); - boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled); + boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, + boolean sourceNatEnabled, Long sourceNatVlanId); + boolean createNetwork(Long zoneId, long accountId, long domainId, Long networkId, String networkName, boolean sourceNatEnabled); boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address); + Long reserveTier1PlacementAndGetPublicVlanId(long zoneId, long accountId, long domainId, Long vpcId, Long networkId); + Long getPublicVlanId(long zoneId, long accountId, long domainId, Long vpcId, Long networkId); + void validatePublicIpVlan(long zoneId, long accountId, long domainId, Long vpcId, Long networkId, long vlanId); String getSegmentId(long domainId, long accountId, long zoneId, Long vpcId, long networkId); + + NsxVpnGatewayResult createVpnGateway(Vpc vpc, String localEndpointIp); + boolean deleteVpnGateway(Vpc vpc); + boolean createVpnConnection(Vpc vpc, String connectionUuid, String peerAddress, String psk, + String ikePolicy, String espPolicy, Long ikeLifetime, Long espLifetime, + boolean dpdEnabled, String ikeVersion, boolean passive, List peerCidrs, + String vtiLocalIp, String vtiPeerIp, int vtiPrefixLength, String localEndpointIp); + boolean deleteVpnConnection(Vpc vpc, String connectionUuid); + boolean updateVpnConnectionState(Vpc vpc, String connectionUuid, boolean enabled); + String getVpnConnectionStatus(Vpc vpc, String connectionUuid); } diff --git a/api/src/main/java/com/cloud/network/nsx/NsxVpnGatewayResult.java b/api/src/main/java/com/cloud/network/nsx/NsxVpnGatewayResult.java new file mode 100644 index 000000000000..c3b71ee2fbcf --- /dev/null +++ b/api/src/main/java/com/cloud/network/nsx/NsxVpnGatewayResult.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.nsx; + +public class NsxVpnGatewayResult { + + private final boolean successful; + private final boolean endpointMayBeInUse; + + public NsxVpnGatewayResult(boolean successful, boolean endpointMayBeInUse) { + this.successful = successful; + this.endpointMayBeInUse = endpointMayBeInUse; + } + + public boolean isSuccessful() { + return successful; + } + + public boolean isEndpointMayBeInUse() { + return endpointMayBeInUse; + } +} diff --git a/api/src/main/java/com/cloud/offering/NetworkOffering.java b/api/src/main/java/com/cloud/offering/NetworkOffering.java index 5000a4f8c626..89463916b93e 100644 --- a/api/src/main/java/com/cloud/offering/NetworkOffering.java +++ b/api/src/main/java/com/cloud/offering/NetworkOffering.java @@ -40,7 +40,9 @@ public enum State { } public enum Detail { - InternalLbProvider, PublicLbProvider, servicepackageuuid, servicepackagedescription, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RelatedNetworkOffering, domainid, zoneid, pvlanType, internetProtocol + InternalLbProvider, PublicLbProvider, servicepackageuuid, servicepackagedescription, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, + NsxIpDiscoveryProfileId, NsxMacDiscoveryProfileId, NsxSegmentSecurityProfileId, + RelatedNetworkOffering, domainid, zoneid, pvlanType, internetProtocol } public enum NetworkMode { diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ac6acdf42516..c5d71432d356 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -135,6 +135,7 @@ public class ApiConstants { public static final String CLUSTER_IDS = "clusterids"; public static final String CLUSTER_NAME = "clustername"; public static final String CLUSTER_TYPE = "clustertype"; + public static final String NETWORK_RULE_OWNERSHIP_STATE = "networkruleownershipstate"; public static final String CN = "cn"; public static final String COMMAND = "command"; public static final String CMD_EVENT_TYPE = "cmdeventtype"; @@ -923,6 +924,7 @@ public class ApiConstants { // NSX public static final String EDGE_CLUSTER = "edgecluster"; public static final String TIER0_GATEWAY = "tier0gateway"; + public static final String PARENT_TIER0_GATEWAY = "parenttier0gateway"; public static final String TRANSPORT_ZONE = "transportzone"; // Tungsten-Fabric diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/NetworkOfferingBaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/NetworkOfferingBaseCmd.java index 9b42be137314..d1b758122644 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/NetworkOfferingBaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/NetworkOfferingBaseCmd.java @@ -166,7 +166,9 @@ public abstract class NetworkOfferingBaseCmd extends BaseCmd { @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.2.0", description = "Network offering details in key/value pairs." + " Supported keys are internallbprovider/publiclbprovider with service provider as a value, and" - + " promiscuousmode/macaddresschanges/forgedtransmits with true/false as value to accept/reject the security settings if available for a nic/portgroup") + + " promiscuousmode/macaddresschanges/forgedtransmits with true/false as value to accept/reject the security settings if available for a nic/portgroup." + + " NSX offerings also support nsxipdiscoveryprofileid, nsxmacdiscoveryprofileid, and nsxsegmentsecurityprofileid." + + " Values are IDs of existing operator-managed NSX profiles to bind to segments created from the offering.") protected Map details; @Parameter(name = ApiConstants.EGRESS_DEFAULT_POLICY, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java index 460b8d642e90..c24d61b7200b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java @@ -32,7 +32,7 @@ import java.util.Map; @APICommand(name = "addObjectStoragePool", description = "Adds a object storage pool", responseObject = ObjectStoreResponse.class, since = "4.19.0", - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = true, responseHasSensitiveInfo = false) public class AddObjectStoragePoolCmd extends BaseCmd { ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/AssociateIPAddrCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/AssociateIPAddrCmd.java index a62f9f316606..1bcbfe0f67a0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/AssociateIPAddrCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/AssociateIPAddrCmd.java @@ -311,7 +311,8 @@ public void create() throws ResourceAllocationException { IpAddress ip = null; if (!isPortable()) { - ip = _networkService.allocateIP(_accountService.getAccount(getEntityOwnerId()), getZoneId(), getNetworkId(), getDisplayIp(), ipAddress); + ip = _networkService.allocateIP(_accountService.getAccount(getEntityOwnerId()), getZoneId(), + getNetworkId(), getVpcId(), getDisplayIp(), ipAddress); } else { ip = _networkService.allocatePortableIP(_accountService.getAccount(getEntityOwnerId()), 1, getZoneId(), getNetworkId(), getVpcId()); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnConnectionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnConnectionCmd.java index b23e6c163020..a1bdf88ed3b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnConnectionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnConnectionCmd.java @@ -29,6 +29,7 @@ import com.cloud.event.EventTypes; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; import com.cloud.user.Account; @APICommand(name = "deleteVpnConnection", description = "Delete site to site VPN connection", responseObject = SuccessResponse.class, entityType = {Site2SiteVpnConnection.class}, @@ -73,6 +74,21 @@ public String getEventType() { return EventTypes.EVENT_S2S_VPN_CONNECTION_DELETE; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + Site2SiteVpnConnection connection = _entityMgr.findById(Site2SiteVpnConnection.class, id); + if (connection == null) { + return null; + } + Site2SiteVpnGateway gateway = _s2sVpnService.getVpnGateway(connection.getVpnGatewayId()); + return gateway == null ? null : gateway.getVpcId(); + } + @Override public void execute() { try { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnGatewayCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnGatewayCmd.java index bfea59a3e6f3..0b82ae6c38e5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnGatewayCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/DeleteVpnGatewayCmd.java @@ -72,6 +72,17 @@ public String getEventType() { return EventTypes.EVENT_S2S_VPN_GATEWAY_DELETE; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + Site2SiteVpnGateway gateway = _entityMgr.findById(Site2SiteVpnGateway.class, id); + return gateway == null ? null : gateway.getVpcId(); + } + @Override public void execute() { boolean result = false; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java index f681c8cce182..198208deb131 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java @@ -30,6 +30,7 @@ import com.cloud.event.EventTypes; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; import com.cloud.user.Account; @APICommand(name = "resetVpnConnection", description = "Reset site to site VPN connection", responseObject = Site2SiteVpnConnectionResponse.class, entityType = {Site2SiteVpnConnection.class}, @@ -91,6 +92,21 @@ public String getEventType() { return EventTypes.EVENT_S2S_VPN_CONNECTION_RESET; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.vpcSyncObject; + } + + @Override + public Long getSyncObjId() { + Site2SiteVpnConnection connection = _entityMgr.findById(Site2SiteVpnConnection.class, id); + if (connection == null) { + return null; + } + Site2SiteVpnGateway gateway = _s2sVpnService.getVpnGateway(connection.getVpnGatewayId()); + return gateway == null ? null : gateway.getVpcId(); + } + @Override public void execute() { try { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java index c7aeb8ba99bf..1669bcb60e6a 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java @@ -20,6 +20,7 @@ import com.cloud.exception.DiscoveryException; import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.context.CallContext; @@ -38,6 +39,8 @@ import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @RunWith(MockitoJUnitRunner.class) @@ -98,4 +101,12 @@ public void testAddObjectStore() throws DiscoveryException { Mockito.verify(storageService, Mockito.times(1)) .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } + + @Test + public void testRequestIsMarkedAsContainingSensitiveInformation() { + APICommand apiCommand = AddObjectStoragePoolCmd.class.getAnnotation(APICommand.class); + + assertNotNull(apiCommand); + assertTrue(apiCommand.requestHasSensitiveInfo()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpn/VpnConnectionLifecycleCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpn/VpnConnectionLifecycleCmdTest.java new file mode 100644 index 000000000000..fa8964feec3a --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpn/VpnConnectionLifecycleCmdTest.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vpn; + +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; +import com.cloud.network.vpn.Site2SiteVpnService; +import com.cloud.utils.db.EntityManager; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class VpnConnectionLifecycleCmdTest { + + private static final Long CONNECTION_ID = 1L; + private static final Long VPN_GATEWAY_ID = 2L; + private static final Long VPC_ID = 3L; + + @Mock + private EntityManager entityManager; + @Mock + private Site2SiteVpnService vpnService; + @Mock + private Site2SiteVpnConnection connection; + @Mock + private Site2SiteVpnGateway gateway; + + private ResetVpnConnectionCmd resetCmd; + private DeleteVpnConnectionCmd deleteCmd; + private DeleteVpnGatewayCmd deleteGatewayCmd; + + @Before + public void setUp() { + resetCmd = new ResetVpnConnectionCmd(); + resetCmd._entityMgr = entityManager; + resetCmd._s2sVpnService = vpnService; + ReflectionTestUtils.setField(resetCmd, "id", CONNECTION_ID); + + deleteCmd = new DeleteVpnConnectionCmd(); + deleteCmd._entityMgr = entityManager; + deleteCmd._s2sVpnService = vpnService; + ReflectionTestUtils.setField(deleteCmd, "id", CONNECTION_ID); + + deleteGatewayCmd = new DeleteVpnGatewayCmd(); + deleteGatewayCmd._entityMgr = entityManager; + ReflectionTestUtils.setField(deleteGatewayCmd, "id", VPN_GATEWAY_ID); + } + + @Test + public void testResetUsesVpcSynchronization() { + configureExistingConnection(); + + assertEquals(BaseAsyncCmd.vpcSyncObject, resetCmd.getSyncObjType()); + assertEquals(VPC_ID, resetCmd.getSyncObjId()); + } + + @Test + public void testDeleteUsesVpcSynchronization() { + configureExistingConnection(); + + assertEquals(BaseAsyncCmd.vpcSyncObject, deleteCmd.getSyncObjType()); + assertEquals(VPC_ID, deleteCmd.getSyncObjId()); + } + + @Test + public void testDeleteGatewayUsesVpcSynchronization() { + when(entityManager.findById(Site2SiteVpnGateway.class, VPN_GATEWAY_ID)).thenReturn(gateway); + when(gateway.getVpcId()).thenReturn(VPC_ID); + + assertEquals(BaseAsyncCmd.vpcSyncObject, deleteGatewayCmd.getSyncObjType()); + assertEquals(VPC_ID, deleteGatewayCmd.getSyncObjId()); + } + + @Test + public void testResetWithMissingConnectionHasNoSynchronizationId() { + when(entityManager.findById(Site2SiteVpnConnection.class, CONNECTION_ID)).thenReturn(null); + + assertNull(resetCmd.getSyncObjId()); + } + + @Test + public void testDeleteWithMissingConnectionHasNoSynchronizationId() { + when(entityManager.findById(Site2SiteVpnConnection.class, CONNECTION_ID)).thenReturn(null); + + assertNull(deleteCmd.getSyncObjId()); + } + + @Test + public void testDeleteGatewayWithMissingGatewayHasNoSynchronizationId() { + when(entityManager.findById(Site2SiteVpnGateway.class, VPN_GATEWAY_ID)).thenReturn(null); + + assertNull(deleteGatewayCmd.getSyncObjId()); + } + + @Test + public void testResetWithMissingGatewayHasNoSynchronizationId() { + when(entityManager.findById(Site2SiteVpnConnection.class, CONNECTION_ID)).thenReturn(connection); + when(connection.getVpnGatewayId()).thenReturn(VPN_GATEWAY_ID); + when(vpnService.getVpnGateway(VPN_GATEWAY_ID)).thenReturn(null); + + assertNull(resetCmd.getSyncObjId()); + } + + @Test + public void testDeleteWithMissingGatewayHasNoSynchronizationId() { + when(entityManager.findById(Site2SiteVpnConnection.class, CONNECTION_ID)).thenReturn(connection); + when(connection.getVpnGatewayId()).thenReturn(VPN_GATEWAY_ID); + when(vpnService.getVpnGateway(VPN_GATEWAY_ID)).thenReturn(null); + + assertNull(deleteCmd.getSyncObjId()); + } + + private void configureExistingConnection() { + when(entityManager.findById(Site2SiteVpnConnection.class, CONNECTION_ID)).thenReturn(connection); + when(connection.getVpnGatewayId()).thenReturn(VPN_GATEWAY_ID); + when(vpnService.getVpnGateway(VPN_GATEWAY_ID)).thenReturn(gateway); + when(gateway.getVpcId()).thenReturn(VPC_ID); + } +} diff --git a/debian/cloudstack-marvin.install b/debian/cloudstack-marvin.install index 5dca252c9ad0..2189d9aa2525 100644 --- a/debian/cloudstack-marvin.install +++ b/debian/cloudstack-marvin.install @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -/usr/share/cloudstack-marvin/Marvin*.tar.gz +/usr/share/cloudstack-marvin/[Mm]arvin*.tar.gz diff --git a/debian/cloudstack-marvin.postinst b/debian/cloudstack-marvin.postinst index a049c1de611f..8adc62544f92 100755 --- a/debian/cloudstack-marvin.postinst +++ b/debian/cloudstack-marvin.postinst @@ -20,4 +20,4 @@ set -e python3 -m pip install --upgrade pip -python3 -m pip install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz +python3 -m pip install --upgrade /usr/share/cloudstack-marvin/[Mm]arvin-*.tar.gz diff --git a/debian/rules b/debian/rules index 327447823308..e67a1279626a 100755 --- a/debian/rules +++ b/debian/rules @@ -177,7 +177,7 @@ override_dh_auto_install: # cloudstack-marvin mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-marvin - cp tools/marvin/dist/Marvin-*.tar.gz $(DESTDIR)/usr/share/$(PACKAGE)-marvin/ + cp tools/marvin/dist/[Mm]arvin-*.tar.gz $(DESTDIR)/usr/share/$(PACKAGE)-marvin/ # cloudstack-integration-tests mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-integration-tests diff --git a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java index 454cb10a2f2b..a69825450cd8 100644 --- a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java @@ -192,9 +192,17 @@ void transferPortableIP(long ipAddrId, long currentNetworkId, long newNetworkId) PublicIp assignDedicateIpAddress(Account owner, Long guestNtwkId, Long vpcId, long dcId, boolean isSourceNat) throws ConcurrentOperationException, InsufficientAddressCapacityException; + PublicIp assignDedicateIpAddressFromNsxVrfPublicRange(Account owner, Long guestNtwkId, Long vpcId, long dcId, + boolean isSourceNat, Long vlanDbId) + throws ConcurrentOperationException, InsufficientAddressCapacityException; + IpAddress allocateIp(Account ipOwner, boolean isSystem, Account caller, User callerId, DataCenter zone, Boolean displayIp, String ipaddress) throws ConcurrentOperationException, ResourceAllocationException, InsufficientAddressCapacityException; + IpAddress allocateIpFromNsxVrfPublicRange(Account ipOwner, boolean isSystem, Account caller, User callerId, DataCenter zone, + Boolean displayIp, String ipaddress, Long vlanDbId) + throws ConcurrentOperationException, ResourceAllocationException, InsufficientAddressCapacityException; + PublicIp assignPublicIpAddressFromVlans(long dcId, Long podId, Account owner, VlanType type, List vlanDbIds, Long networkId, String requestedIp, String requestedGateway, boolean isSystem) throws InsufficientAddressCapacityException; diff --git a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java index 792a3a6b397f..65419cbdf101 100644 --- a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java @@ -57,6 +57,16 @@ public interface VpcManager { true, ConfigKey.Scope.Global, null); + ConfigKey VpcTierDefaultNetworkACL = new ConfigKey<>(String.class, + "vpc.tier.default.network.acl", + ConfigKey.CATEGORY_NETWORK, + "default_allow", + "Network ACL assigned to a VPC tier created without an ACL, either default_allow or default_deny. " + + "Tiers created on a network offering used by the Kubernetes service always get default_allow, " + + "as Kubernetes clusters cannot be deployed on a tier using the default deny ACL", + true, + ConfigKey.Scope.Zone, + null); /** * Returns all the Guest networks that are part of VPC diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDao.java new file mode 100644 index 000000000000..2657a6cd1bf3 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDao.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.List; + +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.utils.db.GenericDao; + +public interface NsxVrfGatewayDao extends GenericDao { + + NsxVrfGatewayVO findByUuid(String uuid); + + NsxVrfGatewayVO findByZoneAndTier0Name(long zoneId, String tier0Name); + + NsxVrfGatewayVO findByPublicVlan(long publicVlanDbId); + + NsxVrfGatewayVO lockByPublicVlan(long publicVlanDbId); + + /** The gateway claimed by this exact account, if any. */ + NsxVrfGatewayVO findByAccount(long zoneId, long accountId); + + /** The gateway claimed by this exact domain, if any — no ancestor walking. */ + NsxVrfGatewayVO findByDomain(long zoneId, long domainId); + + List listByZone(long zoneId); + + /** Pool members in this zone that no tenant has claimed. */ + List listUnclaimed(long zoneId); +} diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDaoImpl.java new file mode 100644 index 000000000000..8cb78dcc9b33 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayDaoImpl.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +@DB() +public class NsxVrfGatewayDaoImpl extends GenericDaoBase implements NsxVrfGatewayDao { + + private final SearchBuilder allFieldsSearch; + private final SearchBuilder unclaimedSearch; + + public NsxVrfGatewayDaoImpl() { + super(); + + allFieldsSearch = createSearchBuilder(); + allFieldsSearch.and("uuid", allFieldsSearch.entity().getUuid(), SearchCriteria.Op.EQ); + allFieldsSearch.and("zone_id", allFieldsSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + allFieldsSearch.and("nsx_tier0_name", allFieldsSearch.entity().getNsxTier0Name(), SearchCriteria.Op.EQ); + allFieldsSearch.and("scope", allFieldsSearch.entity().getScope(), SearchCriteria.Op.EQ); + allFieldsSearch.and("account_id", allFieldsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + allFieldsSearch.and("domain_id", allFieldsSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + allFieldsSearch.and("public_vlan_db_id", allFieldsSearch.entity().getPublicVlanDbId(), SearchCriteria.Op.EQ); + allFieldsSearch.done(); + + unclaimedSearch = createSearchBuilder(); + unclaimedSearch.and("zone_id", unclaimedSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + unclaimedSearch.and("scope", unclaimedSearch.entity().getScope(), SearchCriteria.Op.NULL); + unclaimedSearch.and("account_id", unclaimedSearch.entity().getAccountId(), SearchCriteria.Op.NULL); + unclaimedSearch.and("domain_id", unclaimedSearch.entity().getDomainId(), SearchCriteria.Op.NULL); + unclaimedSearch.done(); + } + + @Override + public NsxVrfGatewayVO findByUuid(String uuid) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("uuid", uuid); + return findOneBy(sc); + } + + @Override + public NsxVrfGatewayVO findByZoneAndTier0Name(long zoneId, String tier0Name) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("nsx_tier0_name", tier0Name); + return findOneBy(sc); + } + + @Override + public NsxVrfGatewayVO findByPublicVlan(long publicVlanDbId) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("public_vlan_db_id", publicVlanDbId); + return findOneBy(sc); + } + + @Override + public NsxVrfGatewayVO lockByPublicVlan(long publicVlanDbId) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("public_vlan_db_id", publicVlanDbId); + return lockOneRandomRow(sc, true); + } + + @Override + public NsxVrfGatewayVO findByAccount(long zoneId, long accountId) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("scope", NsxVrfGatewayVO.Scope.ACCOUNT.name()); + sc.setParameters("account_id", accountId); + return findOneBy(sc); + } + + @Override + public NsxVrfGatewayVO findByDomain(long zoneId, long domainId) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("scope", NsxVrfGatewayVO.Scope.DOMAIN.name()); + sc.setParameters("domain_id", domainId); + return findOneBy(sc); + } + + @Override + public List listByZone(long zoneId) { + SearchCriteria sc = allFieldsSearch.create(); + sc.setParameters("zone_id", zoneId); + return listBy(sc); + } + + @Override + public List listUnclaimed(long zoneId) { + SearchCriteria sc = unclaimedSearch.create(); + sc.setParameters("zone_id", zoneId); + return listBy(sc); + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDao.java new file mode 100644 index 000000000000..2394146c826c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDao.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.List; + +import com.cloud.network.element.NsxVrfGatewayPlacementVO; +import com.cloud.utils.db.GenericDao; + +public interface NsxVrfGatewayPlacementDao extends GenericDao { + + NsxVrfGatewayPlacementVO findByVpcId(long vpcId); + + NsxVrfGatewayPlacementVO findByNetworkId(long networkId); + + long countByGatewayId(long gatewayId); + + List listByZone(long zoneId); +} diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDaoImpl.java new file mode 100644 index 000000000000..c48e0c046149 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/NsxVrfGatewayPlacementDaoImpl.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.network.element.NsxVrfGatewayPlacementVO; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +@DB() +public class NsxVrfGatewayPlacementDaoImpl extends GenericDaoBase + implements NsxVrfGatewayPlacementDao { + + private final SearchBuilder resourceSearch; + private final GenericSearchBuilder gatewayCountSearch; + private final SearchBuilder zoneSearch; + + public NsxVrfGatewayPlacementDaoImpl() { + resourceSearch = createSearchBuilder(); + resourceSearch.and("vpc_id", resourceSearch.entity().getVpcId(), SearchCriteria.Op.EQ); + resourceSearch.and("network_id", resourceSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); + resourceSearch.done(); + + gatewayCountSearch = createSearchBuilder(Long.class); + gatewayCountSearch.select(null, SearchCriteria.Func.COUNT, gatewayCountSearch.entity().getId()); + gatewayCountSearch.and("gateway_id", gatewayCountSearch.entity().getGatewayId(), SearchCriteria.Op.EQ); + gatewayCountSearch.done(); + + zoneSearch = createSearchBuilder(); + zoneSearch.and("zone_id", zoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneSearch.done(); + } + + @Override + public NsxVrfGatewayPlacementVO findByVpcId(long vpcId) { + SearchCriteria sc = resourceSearch.create(); + sc.setParameters("vpc_id", vpcId); + return findOneBy(sc); + } + + @Override + public NsxVrfGatewayPlacementVO findByNetworkId(long networkId) { + SearchCriteria sc = resourceSearch.create(); + sc.setParameters("network_id", networkId); + return findOneBy(sc); + } + + @Override + public long countByGatewayId(long gatewayId) { + SearchCriteria sc = gatewayCountSearch.create(); + sc.setParameters("gateway_id", gatewayId); + List counts = customSearch(sc, null); + return counts.isEmpty() || counts.get(0) == null ? 0L : counts.get(0); + } + + @Override + public List listByZone(long zoneId) { + SearchCriteria sc = zoneSearch.create(); + sc.setParameters("zone_id", zoneId); + return listBy(sc); + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDao.java b/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDao.java index 469ef9a5cc9d..d53b730f123b 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDao.java @@ -18,6 +18,7 @@ import java.util.List; +import com.cloud.network.Site2SiteVpnConnection; import com.cloud.utils.db.GenericDao; public interface Site2SiteVpnConnectionDao extends GenericDao { @@ -27,6 +28,8 @@ public interface Site2SiteVpnConnectionDao extends GenericDao listByVpcId(long vpcId); + List listByStates(Site2SiteVpnConnection.State... states); + Site2SiteVpnConnectionVO findByVpnGatewayIdAndCustomerGatewayId(long vpnId, long customerId); Site2SiteVpnConnectionVO findByCustomerGatewayId(long customerId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImpl.java index f9c5ce089645..701e207ecaa0 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImpl.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; +import com.cloud.network.Site2SiteVpnConnection; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; @@ -39,6 +40,7 @@ public class Site2SiteVpnConnectionDaoImpl extends GenericDaoBase AllFieldsSearch; private SearchBuilder VpcSearch; private SearchBuilder VpnGatewaySearch; + private SearchBuilder StateSearch; public Site2SiteVpnConnectionDaoImpl() { } @@ -55,6 +57,10 @@ protected void init() { VpnGatewaySearch.and("vpcId", VpnGatewaySearch.entity().getVpcId(), SearchCriteria.Op.EQ); VpcSearch.join("vpnGatewaySearch", VpnGatewaySearch, VpnGatewaySearch.entity().getId(), VpcSearch.entity().getVpnGatewayId(), JoinType.INNER); VpcSearch.done(); + + StateSearch = createSearchBuilder(); + StateSearch.and("state", StateSearch.entity().getState(), SearchCriteria.Op.IN); + StateSearch.done(); } @Override @@ -78,6 +84,13 @@ public List listByVpcId(long vpcId) { return listBy(sc); } + @Override + public List listByStates(Site2SiteVpnConnection.State... states) { + SearchCriteria sc = StateSearch.create(); + sc.setParameters("state", (Object[]) states); + return listBy(sc); + } + @Override public Site2SiteVpnConnectionVO findByVpnGatewayIdAndCustomerGatewayId(long vpnId, long customerId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayPlacementVO.java b/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayPlacementVO.java new file mode 100644 index 000000000000..755e73748892 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayPlacementVO.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.element; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "nsx_vrf_gateway_placements") +public class NsxVrfGatewayPlacementVO implements InternalIdentity { + + public enum State { + PENDING_CREATE, + ACTIVE, + PENDING_DELETE, + FAILED + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "gateway_id") + private Long gatewayId; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "vpc_id") + private Long vpcId; + + @Column(name = "network_id") + private Long networkId; + + @Column(name = "tier0_name") + private String tier0Name; + + @Column(name = "state") + private String state; + + @Column(name = "created") + private Date created; + + @Column(name = "updated") + private Date updated; + + protected NsxVrfGatewayPlacementVO() { + } + + public NsxVrfGatewayPlacementVO(Long gatewayId, long zoneId, long domainId, long accountId, + Long vpcId, Long networkId, String tier0Name) { + if ((vpcId == null) == (networkId == null)) { + throw new IllegalArgumentException("Exactly one of vpcId or networkId is required"); + } + this.gatewayId = gatewayId; + this.zoneId = zoneId; + this.domainId = domainId; + this.accountId = accountId; + this.vpcId = vpcId; + this.networkId = networkId; + this.tier0Name = tier0Name; + this.state = State.PENDING_CREATE.name(); + this.created = new Date(); + this.updated = this.created; + } + + @Override + public long getId() { + return id; + } + + public Long getGatewayId() { + return gatewayId; + } + + public long getZoneId() { + return zoneId; + } + + public long getDomainId() { + return domainId; + } + + public long getAccountId() { + return accountId; + } + + public Long getVpcId() { + return vpcId; + } + + public Long getNetworkId() { + return networkId; + } + + public String getTier0Name() { + return tier0Name; + } + + public String getState() { + return state; + } + + public void setState(State state) { + this.state = state.name(); + this.updated = new Date(); + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayVO.java b/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayVO.java new file mode 100644 index 000000000000..36773673edc4 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/element/NsxVrfGatewayVO.java @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.element; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.InternalIdentity; + +/** + * A VRF (or dedicated) Tier-0 gateway that an operator has staged in NSX and registered + * with CloudStack, so that a tenant's Tier-1 gateways attach to it instead of to the + * zone-wide Tier-0 recorded in {@code nsx_providers}. + * + * CloudStack never creates or deletes the gateway in NSX: the uplink interfaces and BGP + * peerings a Tier-0 needs are provisioned with the operator's physical network and cannot + * be automated from here. A row is a claim on something that already exists. + * + * A row with neither {@code accountId} nor {@code domainId} is an unclaimed pool member. + */ +@Entity +@Table(name = "nsx_vrf_gateways") +public class NsxVrfGatewayVO implements InternalIdentity { + + /** How a staged gateway is matched to a tenant. */ + public enum Scope { + ACCOUNT, DOMAIN + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "nsx_tier0_name") + private String nsxTier0Name; + + @Column(name = "edge_cluster") + private String edgeCluster; + + @Column(name = "parent_tier0") + private String parentTier0; + + @Column(name = "scope") + private String scope; + + @Column(name = "domain_id") + private Long domainId; + + @Column(name = "account_id") + private Long accountId; + + @Column(name = "public_vlan_db_id") + private Long publicVlanDbId; + + @Column(name = "created") + private Date created; + + @Column(name = "removed") + private Date removed; + + public NsxVrfGatewayVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public NsxVrfGatewayVO(long zoneId, String nsxTier0Name, String edgeCluster, String parentTier0) { + this(); + this.zoneId = zoneId; + this.nsxTier0Name = nsxTier0Name; + this.edgeCluster = edgeCluster; + this.parentTier0 = parentTier0; + this.created = new Date(); + } + + @Override + public long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public long getZoneId() { + return zoneId; + } + + public void setZoneId(long zoneId) { + this.zoneId = zoneId; + } + + public String getNsxTier0Name() { + return nsxTier0Name; + } + + public void setNsxTier0Name(String nsxTier0Name) { + this.nsxTier0Name = nsxTier0Name; + } + + public String getEdgeCluster() { + return edgeCluster; + } + + public void setEdgeCluster(String edgeCluster) { + this.edgeCluster = edgeCluster; + } + + public String getParentTier0() { + return parentTier0; + } + + public void setParentTier0(String parentTier0) { + this.parentTier0 = parentTier0; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getPublicVlanDbId() { + return publicVlanDbId; + } + + public void setPublicVlanDbId(Long publicVlanDbId) { + this.publicVlanDbId = publicVlanDbId; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + /** True when no tenant has claimed this gateway yet. */ + public boolean isUnclaimed() { + return scope == null && accountId == null && domainId == null; + } + + // GenericDao update tracking requires assignment changes to use these setters. + + @Override + public String toString() { + return String.format("NsxVrfGateway {id: %d, uuid: %s, tier0: %s, edgeCluster: %s, scope: %s}", + id, uuid, nsxTier0Name, edgeCluster, scope); + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/dao/NetworkACLItemDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/vpc/dao/NetworkACLItemDaoImpl.java index 925515f6f4ab..5fbfed4e9dde 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/dao/NetworkACLItemDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/dao/NetworkACLItemDaoImpl.java @@ -86,7 +86,9 @@ protected NetworkACLItemDaoImpl() { @Override public NetworkACLItemVO findById(Long id) { NetworkACLItemVO item = super.findById(id); - loadCidrs(item); + if (item != null) { + loadCidrs(item); + } return item; } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDao.java b/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDao.java index 68798b33b77a..f1a57a8eb13e 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDao.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDao.java @@ -35,6 +35,8 @@ public interface VpcDao extends GenericDao { List listByAccountId(long accountId); + List listByZone(long zoneId); + List listInactiveVpcs(); long countByAccountId(long accountId); diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDaoImpl.java index cecdce1aba42..375c3da5e750 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/dao/VpcDaoImpl.java @@ -63,6 +63,7 @@ protected VpcDaoImpl() { AllFieldsSearch.and("id", AllFieldsSearch.entity().getId(), Op.EQ); AllFieldsSearch.and("state", AllFieldsSearch.entity().getState(), Op.EQ); AllFieldsSearch.and("accountId", AllFieldsSearch.entity().getAccountId(), Op.EQ); + AllFieldsSearch.and("zoneId", AllFieldsSearch.entity().getZoneId(), Op.EQ); AllFieldsSearch.done(); CountByAccountId = createSearchBuilder(Long.class); @@ -95,6 +96,13 @@ public List listByAccountId(long accountId) { return listBy(sc, null); } + @Override + public List listByZone(long zoneId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("zoneId", zoneId); + return listBy(sc, null); + } + @Override public List listInactiveVpcs() { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 932db538f30b..7266eacc03f7 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -140,6 +140,8 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index ab5ac7b2b875..0f6a7541ba35 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -646,3 +646,178 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule', 'isolated', 'TINYI UPDATE `cloud`.`configuration` SET `value`=CONCAT(`value`, ', backupValidationCommandTimeout, backupValidationScreenshotWait, backupValidationBootTimeout') WHERE `name`='user.vm.readonly.details' AND `value` IS NOT NULL; + +-- NSX per-tenant VRF gateways. +-- One row per operator-staged VRF (or dedicated) Tier-0. A row with no account_id and +-- no domain_id is an unclaimed member of the pool; assigning it to a tenant fills them. +-- CloudStack never creates or deletes the gateway in NSX, it only records and claims. +CREATE TABLE IF NOT EXISTS `cloud`.`nsx_vrf_gateways` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) NOT NULL, + `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID', + `nsx_tier0_name` varchar(255) NOT NULL COMMENT 'VRF Tier-0, or a dedicated Tier-0, as named in NSX', + `edge_cluster` varchar(255) NOT NULL COMMENT 'Edge cluster this Tier-0 lives on; may differ from the zone default', + `parent_tier0` varchar(255) COMMENT 'Parent Tier-0 for a VRF gateway; NULL for a dedicated Tier-0', + `scope` varchar(16) COMMENT 'ACCOUNT or DOMAIN; NULL while unclaimed', + `domain_id` bigint unsigned COMMENT 'Owning domain when scope = DOMAIN', + `account_id` bigint unsigned COMMENT 'Owning account when scope = ACCOUNT', + `public_vlan_db_id` bigint unsigned COMMENT 'Dedicated public IP range this Tier-0 advertises', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + CONSTRAINT `fk_nsx_vrf_gateways__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_nsx_vrf_gateways__account_id` FOREIGN KEY (`account_id`) REFERENCES `account`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_nsx_vrf_gateways__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_nsx_vrf_gateways__public_vlan_db_id` FOREIGN KEY (`public_vlan_db_id`) REFERENCES `vlan`(`id`) ON DELETE RESTRICT, + UNIQUE KEY `uk_nsx_vrf_gateways__uuid` (`uuid`), + UNIQUE KEY `uk_nsx_vrf_gateways__zone_tier0` (`zone_id`, `nsx_tier0_name`), + UNIQUE KEY `uk_nsx_vrf_gateways__zone_account` (`zone_id`, `account_id`), + UNIQUE KEY `uk_nsx_vrf_gateways__zone_domain` (`zone_id`, `domain_id`), + UNIQUE KEY `uk_nsx_vrf_gateways__public_vlan` (`public_vlan_db_id`), + INDEX `i_nsx_vrf_gateways__zone_id`(`zone_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Normalize an existing table before applying ownership constraints; CREATE TABLE IF NOT +-- EXISTS does not alter an earlier table definition. +UPDATE `cloud`.`nsx_vrf_gateways` SET `uuid` = UUID() WHERE `uuid` IS NULL OR `uuid` = ''; + +DROP PROCEDURE IF EXISTS `cloud`.`VALIDATE_NSX_VRF_GATEWAY_OWNERSHIP`; +CREATE PROCEDURE `cloud`.`VALIDATE_NSX_VRF_GATEWAY_OWNERSHIP`() +BEGIN + IF EXISTS ( + SELECT 1 FROM `cloud`.`nsx_vrf_gateways` + WHERE (`scope` = 'ACCOUNT' AND (`account_id` IS NULL OR `domain_id` IS NOT NULL)) + OR (`scope` = 'DOMAIN' AND (`domain_id` IS NULL OR `account_id` IS NOT NULL)) + OR (`scope` IS NULL AND (`account_id` IS NOT NULL OR `domain_id` IS NOT NULL)) + OR (`scope` IS NOT NULL AND `scope` NOT IN ('ACCOUNT', 'DOMAIN')) + ) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Invalid NSX VRF ownership state' +; END IF +; END; +CALL `cloud`.`VALIDATE_NSX_VRF_GATEWAY_OWNERSHIP`(); +DROP PROCEDURE `cloud`.`VALIDATE_NSX_VRF_GATEWAY_OWNERSHIP`; + +ALTER TABLE `cloud`.`nsx_vrf_gateways` MODIFY COLUMN `uuid` varchar(40) NOT NULL; + +CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_nsx_vrf_gateways__account_id', 'cloud.nsx_vrf_gateways'); +CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_nsx_vrf_gateways__domain_id', 'cloud.nsx_vrf_gateways'); +CALL `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`('uk_nsx_vrf_gateways__uuid', 'cloud.nsx_vrf_gateways', '(`uuid`)'); +CALL `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`('uk_nsx_vrf_gateways__zone_tier0', 'cloud.nsx_vrf_gateways', '(`zone_id`, `nsx_tier0_name`)'); +CALL `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`('uk_nsx_vrf_gateways__zone_account', 'cloud.nsx_vrf_gateways', '(`zone_id`, `account_id`)'); +CALL `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`('uk_nsx_vrf_gateways__zone_domain', 'cloud.nsx_vrf_gateways', '(`zone_id`, `domain_id`)'); +CALL `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`('uk_nsx_vrf_gateways__public_vlan', 'cloud.nsx_vrf_gateways', '(`public_vlan_db_id`)'); + +CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.nsx_vrf_gateways', 'fk_nsx_vrf_gateways__account_id'); +CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.nsx_vrf_gateways', 'fk_nsx_vrf_gateways__domain_id'); +CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.nsx_vrf_gateways', 'fk_nsx_vrf_gateways__public_vlan_db_id'); +ALTER TABLE `cloud`.`nsx_vrf_gateways` + ADD CONSTRAINT `fk_nsx_vrf_gateways__account_id` FOREIGN KEY (`account_id`) REFERENCES `account`(`id`) ON DELETE RESTRICT, + ADD CONSTRAINT `fk_nsx_vrf_gateways__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`) ON DELETE RESTRICT, + ADD CONSTRAINT `fk_nsx_vrf_gateways__public_vlan_db_id` FOREIGN KEY (`public_vlan_db_id`) REFERENCES `vlan`(`id`) ON DELETE RESTRICT; + +-- Durable record of the Tier-0 selected for each CloudStack-owned NSX Tier-1. +-- Shared Tier-0 placements deliberately have gateway_id NULL: recording them prevents a +-- later VRF assignment from silently changing the parent on an idempotent create retry. +CREATE TABLE IF NOT EXISTS `cloud`.`nsx_vrf_gateway_placements` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `gateway_id` bigint unsigned COMMENT 'Registered VRF gateway, NULL for the zone shared Tier-0', + `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID', + `domain_id` bigint unsigned NOT NULL COMMENT 'Owning domain at placement time', + `account_id` bigint unsigned NOT NULL COMMENT 'Owning account at placement time', + `vpc_id` bigint unsigned COMMENT 'VPC whose Tier-1 was placed', + `network_id` bigint unsigned COMMENT 'Isolated network whose Tier-1 was placed', + `tier0_name` varchar(255) NOT NULL COMMENT 'Resolved Tier-0 name at placement time', + `state` varchar(32) NOT NULL COMMENT 'PENDING_CREATE, ACTIVE, PENDING_DELETE, or FAILED', + `created` datetime NOT NULL, + `updated` datetime NOT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_nsx_vrf_gateway_placements__gateway_id` FOREIGN KEY (`gateway_id`) REFERENCES `nsx_vrf_gateways`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_nsx_vrf_gateway_placements__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_nsx_vrf_gateway_placements__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_nsx_vrf_gateway_placements__account_id` FOREIGN KEY (`account_id`) REFERENCES `account`(`id`) ON DELETE RESTRICT, + CONSTRAINT `fk_nsx_vrf_gateway_placements__vpc_id` FOREIGN KEY (`vpc_id`) REFERENCES `vpc`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_nsx_vrf_gateway_placements__network_id` FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`) ON DELETE CASCADE, + UNIQUE KEY `uk_nsx_vrf_gateway_placements__vpc_id` (`vpc_id`), + UNIQUE KEY `uk_nsx_vrf_gateway_placements__network_id` (`network_id`), + INDEX `i_nsx_vrf_gateway_placements__gateway_id` (`gateway_id`), + INDEX `i_nsx_vrf_gateway_placements__zone_id` (`zone_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Before per-tenant VRF placement existed, every CloudStack-created NSX Tier-1 used the +-- active NSX provider's zone-wide Tier-0. Preserve that parent deterministically so a +-- later tenant assignment cannot re-parent an existing Tier-1 on an idempotent retry. +INSERT INTO `cloud`.`nsx_vrf_gateway_placements` + (`gateway_id`, `zone_id`, `domain_id`, `account_id`, `vpc_id`, `network_id`, + `tier0_name`, `state`, `created`, `updated`) +SELECT NULL, v.`zone_id`, v.`domain_id`, v.`account_id`, v.`id`, NULL, + np.`tier0_gateway`, 'ACTIVE', v.`created`, NOW() +FROM `cloud`.`vpc` v +JOIN `cloud`.`nsx_providers` np + ON np.`zone_id` = v.`zone_id` AND np.`removed` IS NULL +WHERE v.`removed` IS NULL + AND np.`tier0_gateway` IS NOT NULL + AND np.`tier0_gateway` <> '' + AND EXISTS ( + SELECT 1 + FROM `cloud`.`vpc_offering_service_map` vosm + WHERE vosm.`vpc_offering_id` = v.`vpc_offering_id` + AND vosm.`provider` = 'Nsx' + AND vosm.`service` IN ('SourceNat', 'Gateway') + ) + AND NOT EXISTS ( + SELECT 1 FROM `cloud`.`nsx_vrf_gateway_placements` p WHERE p.`vpc_id` = v.`id` + ); + +INSERT INTO `cloud`.`nsx_vrf_gateway_placements` + (`gateway_id`, `zone_id`, `domain_id`, `account_id`, `vpc_id`, `network_id`, + `tier0_name`, `state`, `created`, `updated`) +SELECT NULL, n.`data_center_id`, n.`domain_id`, n.`account_id`, NULL, n.`id`, + np.`tier0_gateway`, 'ACTIVE', n.`created`, NOW() +FROM `cloud`.`networks` n +JOIN `cloud`.`nsx_providers` np + ON np.`zone_id` = n.`data_center_id` AND np.`removed` IS NULL +WHERE n.`removed` IS NULL + AND n.`vpc_id` IS NULL + AND n.`broadcast_domain_type` = 'NSX' + AND np.`tier0_gateway` IS NOT NULL + AND np.`tier0_gateway` <> '' + AND NOT EXISTS ( + SELECT 1 FROM `cloud`.`nsx_vrf_gateway_placements` p WHERE p.`network_id` = n.`id` + ); + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'network_rule_ownership_state', + 'varchar(32) NOT NULL DEFAULT ''LEGACY_UNMANAGED'' COMMENT ''CKS managed network-rule ownership migration state'' AFTER `cluster_type`'); + +-- CKS-managed network-rule ownership. Firewall, port-forwarding, and load-balancer +-- rules share firewall_rules; VPC ACL items have their own table. Logical roles are +-- stable desired identities while lifecycle_state preserves ownership across +-- asynchronous provider application and deletion. +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_firewall_rule_map` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `cluster_id` bigint unsigned NOT NULL COMMENT 'Kubernetes cluster that owns the rule', + `firewall_rule_id` bigint unsigned NOT NULL COMMENT 'CloudStack-managed firewall, port-forwarding, or load-balancer rule', + `logical_role` varchar(128) NOT NULL COMMENT 'Stable desired role within the Kubernetes cluster', + `lifecycle_state` varchar(32) NOT NULL COMMENT 'PENDING_APPLY, ACTIVE, or PENDING_DELETE', + `created` datetime NOT NULL, + `updated` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_kubernetes_cluster_firewall_rule_map__rule_id` (`firewall_rule_id`), + UNIQUE KEY `uk_kubernetes_cluster_firewall_rule_map__cluster_role` (`cluster_id`, `logical_role`), + CONSTRAINT `fk_kubernetes_cluster_firewall_rule_map__rule_id` FOREIGN KEY (`firewall_rule_id`) REFERENCES `firewall_rules` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_cluster_firewall_rule_map__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_network_acl_item_map` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `cluster_id` bigint unsigned NOT NULL COMMENT 'Kubernetes cluster that owns the ACL item', + `network_acl_item_id` bigint unsigned NOT NULL COMMENT 'CloudStack-managed VPC network ACL item', + `logical_role` varchar(128) NOT NULL COMMENT 'Stable desired role within the Kubernetes cluster', + `lifecycle_state` varchar(32) NOT NULL COMMENT 'PENDING_APPLY, ACTIVE, or PENDING_DELETE', + `created` datetime NOT NULL, + `updated` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_kubernetes_cluster_network_acl_item_map__item_id` (`network_acl_item_id`), + UNIQUE KEY `uk_kubernetes_cluster_network_acl_item_map__cluster_role` (`cluster_id`, `logical_role`), + CONSTRAINT `fk_kubernetes_cluster_network_acl_item_map__item_id` FOREIGN KEY (`network_acl_item_id`) REFERENCES `network_acl_item` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_cluster_network_acl_item_map__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/test/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImplTest.java new file mode 100644 index 000000000000..ffec95ab0719 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/dao/Site2SiteVpnConnectionDaoImplTest.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.List; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class Site2SiteVpnConnectionDaoImplTest { + + @Spy + private Site2SiteVpnConnectionDaoImpl dao; + @Mock + private SearchBuilder stateSearch; + @Mock + private SearchCriteria searchCriteria; + + @Test + public void testListByStatesUsesStateSearchCriteria() { + ReflectionTestUtils.setField(dao, "StateSearch", stateSearch); + when(stateSearch.create()).thenReturn(searchCriteria); + doReturn(List.of()).when(dao).listBy(searchCriteria); + Site2SiteVpnConnection.State[] states = { + Site2SiteVpnConnection.State.Pending, + Site2SiteVpnConnection.State.Connecting, + Site2SiteVpnConnection.State.Connected, + Site2SiteVpnConnection.State.Disconnected + }; + + dao.listByStates(states); + + verify(searchCriteria).setParameters("state", (Object[]) states); + verify(dao).listBy(searchCriteria); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index d11a0fc3dea5..dbf3cb522bbc 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -26,4 +26,6 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_RECONCILE = "KUBERNETES.CLUSTER.NETWORK.RULES.RECONCILE"; + public static final String EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_ADOPT = "KUBERNETES.CLUSTER.NETWORK.RULES.ADOPT"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterFirewallRuleMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterFirewallRuleMapVO.java new file mode 100644 index 000000000000..1ee7d345d5d6 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterFirewallRuleMapVO.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import java.util.Date; + +import org.apache.cloudstack.api.InternalIdentity; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "kubernetes_cluster_firewall_rule_map") +public class KubernetesClusterFirewallRuleMapVO implements InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "cluster_id") + private long clusterId; + + @Column(name = "firewall_rule_id") + private long firewallRuleId; + + @Column(name = "logical_role") + private String logicalRole; + + @Enumerated(EnumType.STRING) + @Column(name = "lifecycle_state") + private KubernetesClusterNetworkRuleLifecycleState lifecycleState; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = "updated") + private Date updated; + + public KubernetesClusterFirewallRuleMapVO() { + } + + public KubernetesClusterFirewallRuleMapVO(long clusterId, long firewallRuleId, String logicalRole, + KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + this.clusterId = clusterId; + this.firewallRuleId = firewallRuleId; + this.logicalRole = logicalRole; + this.lifecycleState = lifecycleState; + this.updated = new Date(); + } + + @Override + public long getId() { + return id; + } + + public long getFirewallRuleId() { + return firewallRuleId; + } + + public long getClusterId() { + return clusterId; + } + + public String getLogicalRole() { + return logicalRole; + } + + public KubernetesClusterNetworkRuleLifecycleState getLifecycleState() { + return lifecycleState; + } + + public void setLifecycleState(KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + this.lifecycleState = lifecycleState; + this.updated = new Date(); + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index aa5ddf0cd006..6ea5e1805e9c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -67,6 +67,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.VMDetails; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; @@ -86,6 +87,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ReconcileKubernetesClusterNetworkRulesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -112,6 +114,7 @@ import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.cluster.AdoptKubernetesClusterNetworkRulesCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.KubernetesUserVmResponse; @@ -122,6 +125,8 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.commons.beanutils.BeanUtils; @@ -132,7 +137,9 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; +import com.cloud.api.ApiAsyncJobDispatcher; import com.cloud.api.ApiDBUtils; +import com.cloud.api.ApiGsonHelper; import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; @@ -152,6 +159,7 @@ import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.deploy.DeployDestination; import com.cloud.event.ActionEvent; +import com.cloud.event.ActionEventUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; @@ -176,6 +184,8 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNetworkACLItemMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; @@ -320,6 +330,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public KubernetesClusterDetailsDao kubernetesClusterDetailsDao; @Inject + public KubernetesClusterFirewallRuleMapDao kubernetesClusterFirewallRuleMapDao; + @Inject + public KubernetesClusterNetworkACLItemMapDao kubernetesClusterNetworkACLItemMapDao; + @Inject public KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; @Inject public KubernetesSupportedVersionDao kubernetesSupportedVersionDao; @@ -411,6 +425,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne RoleService roleService; @Inject ResourceLimitService resourceLimitService; + @Inject + AsyncJobManager asyncJobManager; + @Inject + ApiAsyncJobDispatcher apiAsyncJobDispatcher; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -608,10 +626,11 @@ private void validateIsolatedNetwork(Network network, int clusterTotalNodeCount) } protected void validateVpcTier(Network network) { - if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules - return; + Long networkAclId = network.getNetworkACLId(); + if (networkAclId == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s can not be used for Kubernetes cluster as it does not have a network ACL attached. Attach a network ACL allowing the required traffic to the VPC tier and retry", network.getUuid())); } - if (network.getNetworkACLId() == NetworkACL.DEFAULT_DENY) { + if (Objects.equals(networkAclId, NetworkACL.DEFAULT_DENY)) { throw new InvalidParameterValueException(String.format("Network ID: %s can not be used for Kubernetes cluster as it uses default deny ACL", network.getUuid())); } } @@ -943,6 +962,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setMinSize(kubernetesCluster.getMinSize()); response.setMaxSize(kubernetesCluster.getMaxSize()); response.setClusterType(kubernetesCluster.getClusterType()); + response.setNetworkRuleOwnershipState(kubernetesCluster.getNetworkRuleOwnershipState()); response.setCsiEnabled(kubernetesCluster.isCsiEnabled()); response.setCreated(kubernetesCluster.getCreated()); setNodeTypeAffinityGroupResponse(response, kubernetesCluster.getId()); @@ -1062,7 +1082,9 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), - BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) + BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class), + BaseCmd.getCommandNameByClass(AdoptKubernetesClusterNetworkRulesCmd.class) ).contains(cmdName); case ExternalManaged: return Arrays.asList( @@ -2263,6 +2285,10 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou } validateKubernetesClusterScaleParameters(cmd); KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + if ((cmd.getClusterSize() != null && cmd.getClusterSize() != kubernetesCluster.getNodeCount()) + || CollectionUtils.isNotEmpty(cmd.getNodeIds())) { + validateNetworkRuleOwnershipForTopologyMutation(kubernetesCluster); + } Map nodeToOfferingMap = createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); String[] keys = getServiceUserKeys(kubernetesCluster); @@ -2280,6 +2306,118 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou return scaleWorker.scaleCluster(); } + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_RECONCILE, + eventDescription = "reconciling Kubernetes cluster network rules", async = true) + public boolean reconcileKubernetesClusterNetworkRules(ReconcileKubernetesClusterNetworkRulesCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO cluster = kubernetesClusterDao.findById(cmd.getId()); + if (cluster == null || cluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, cluster); + if (!isCommandSupported(cluster, cmd.getActualCommandName())) { + throw new InvalidParameterValueException(String.format("Network rule reconciliation is not supported for an externally managed cluster (%s)", cluster.getName())); + } + boolean alertRecovery = KubernetesCluster.State.Recovering.equals(cluster.getState()) + && Objects.equals(CallContext.current().getCallingUserId(), User.UID_SYSTEM); + if (!alertRecovery && !KubernetesCluster.State.Running.equals(cluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s must be running to reconcile its network rules", cluster.getName())); + } + if (!alertRecovery && !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(cluster.getNetworkRuleOwnershipState())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s network-rule ownership must be adopted before reconciliation", + cluster.getName())); + } + + KubernetesClusterStartWorker worker = createKubernetesClusterStartWorker(cluster); + if (!alertRecovery) { + return worker.reconcileKubernetesClusterNetworkRules(); + } + try { + boolean recovered = worker.reconcileAlertCluster(); + if (!recovered) { + stateTransitTo(cluster.getId(), KubernetesCluster.Event.OperationFailed); + } + return recovered; + } catch (RuntimeException e) { + stateTransitTo(cluster.getId(), KubernetesCluster.Event.OperationFailed); + throw e; + } + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_ADOPT, + eventDescription = "adopting Kubernetes cluster network-rule ownership", async = true) + public boolean adoptKubernetesClusterNetworkRules(AdoptKubernetesClusterNetworkRulesCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + Account caller = CallContext.current().getCallingAccount(); + if (caller == null || !accountManager.isRootAdmin(caller.getId())) { + throw new PermissionDeniedException("Only a root administrator can adopt Kubernetes cluster network-rule ownership"); + } + KubernetesClusterVO cluster = kubernetesClusterDao.findById(cmd.getId()); + if (cluster == null || cluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + if (!KubernetesCluster.ClusterType.CloudManaged.equals(cluster.getClusterType())) { + throw new InvalidParameterValueException(String.format( + "Network-rule ownership adoption is not supported for externally managed cluster %s", cluster.getName())); + } + Set stableStates = EnumSet.of(KubernetesCluster.State.Running, + KubernetesCluster.State.Stopped, KubernetesCluster.State.Alert, KubernetesCluster.State.Error); + if (!stableStates.contains(cluster.getState())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s must be in a stable state before network-rule ownership can be adopted", cluster.getName())); + } + + KubernetesClusterStartWorker worker = createKubernetesClusterStartWorker(cluster); + return worker.adoptKubernetesClusterNetworkRules(cmd.getRuleSpecs()); + } + + protected KubernetesClusterStartWorker createKubernetesClusterStartWorker(KubernetesCluster cluster) { + return ComponentContext.inject(new KubernetesClusterStartWorker(cluster, this)); + } + + protected long scheduleAlertClusterReconciliation(KubernetesClusterVO cluster) { + Long networkId = cluster.getNetworkId(); + if (networkId == null) { + throw new CloudRuntimeException(String.format( + "Kubernetes cluster %s has no network to synchronize alert reconciliation on", cluster.getName())); + } + + Map params = new HashMap<>(); + params.put(ApiConstants.ID, String.valueOf(cluster.getId())); + params.put("ctxUserId", String.valueOf(User.UID_SYSTEM)); + params.put("ctxAccountId", String.valueOf(cluster.getAccountId())); + params.put(ApiConstants.CTX_START_EVENT_ID, String.valueOf(createAlertClusterReconciliationEvent(cluster))); + + AsyncJobVO job = new AsyncJobVO("", User.UID_SYSTEM, cluster.getAccountId(), + ReconcileKubernetesClusterNetworkRulesCmd.class.getName(), + ApiGsonHelper.getBuilder().create().toJson(params), cluster.getId(), + ApiCommandResourceType.KubernetesCluster.toString(), null); + job.setDispatcher(apiAsyncJobDispatcher.getName()); + + // Submit directly to the lifecycle network queue. The resumed API job executes the + // command body and must not submit another nested reconciliation job for the same queue. + long jobId = asyncJobManager.submitAsyncJob(job, BaseAsyncCmd.networkSyncObject, networkId); + if (jobId == 0L) { + throw new CloudRuntimeException(String.format( + "Failed to schedule alert reconciliation for Kubernetes cluster %s", cluster.getName())); + } + return jobId; + } + + protected long createAlertClusterReconciliationEvent(KubernetesClusterVO cluster) { + return ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, cluster.getAccountId(), + KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_RECONCILE, + String.format("Reconciling alert Kubernetes cluster %s", cluster.getUuid()), cluster.getId(), + ApiCommandResourceType.KubernetesCluster.toString(), true, 0L); + } + /** * Creates a map for the requested node type service offering * For the node type DEFAULT: Every node is scaled to the same offering @@ -2464,6 +2602,8 @@ public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + validateAccessToClusterAndNodes(kubernetesCluster, cmd.getNodeIds()); + validateNetworkRuleOwnershipForTopologyMutation(kubernetesCluster); long networkId = kubernetesCluster.getNetworkId(); NetworkVO networkVO = networkDao.findById(networkId); List validNodeIds = validateNodes(cmd.getNodeIds(), networkId, networkVO.getName(), kubernetesCluster, false); @@ -2482,6 +2622,8 @@ public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesCluster logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + validateAccessToClusterAndNodes(kubernetesCluster, cmd.getNodeIds()); + validateNetworkRuleOwnershipForTopologyMutation(kubernetesCluster); List validNodeIds = validateNodes(cmd.getNodeIds(), null, null, kubernetesCluster, true); if (validNodeIds.isEmpty()) { throw new CloudRuntimeException("No valid nodes found to be removed from the Kubernetes cluster"); @@ -2499,6 +2641,35 @@ private KubernetesClusterVO validateCluster(long clusterId) { return kubernetesCluster; } + protected void validateAccessToClusterAndNodes(KubernetesClusterVO kubernetesCluster, List nodeIds) { + Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + for (Long nodeId : nodeIds) { + VMInstanceVO node = vmInstanceDao.findById(nodeId); + if (node == null) { + throw new InvalidParameterValueException(String.format("Invalid node ID %d specified", nodeId)); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, node); + } + } + + protected void validateNetworkRuleOwnershipForTopologyMutation(KubernetesClusterVO kubernetesCluster) { + if (!KubernetesCluster.ClusterType.CloudManaged.equals(kubernetesCluster.getClusterType())) { + return; + } + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + throw new InvalidParameterValueException(String.format("Network for Kubernetes cluster %s cannot be found", + kubernetesCluster.getName())); + } + if (!isDirectAccess(network) + && !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(kubernetesCluster.getNetworkRuleOwnershipState())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s network-rule ownership must be adopted before changing its node topology", + kubernetesCluster.getName())); + } + } + private List validateNodes(List nodeIds, Long networkId, String networkName, KubernetesCluster cluster, boolean removeNodes) { List validNodeIds = new ArrayList<>(nodeIds); for (Long id : nodeIds) { @@ -2759,6 +2930,8 @@ public List> getCommands() { cmdList.add(AddNodesToKubernetesClusterCmd.class); cmdList.add(RemoveNodesFromKubernetesClusterCmd.class); cmdList.add(UpdateKubernetesClusterAffinityGroupCmd.class); + cmdList.add(ReconcileKubernetesClusterNetworkRulesCmd.class); + cmdList.add(AdoptKubernetesClusterNetworkRulesCmd.class); return cmdList; } @@ -2878,10 +3051,19 @@ public void reallyRun() { } try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - KubernetesClusterStartWorker startWorker = - new KubernetesClusterStartWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); - startWorker = ComponentContext.inject(startWorker); - startWorker.reconcileAlertCluster(); + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested)) { + logger.debug("Kubernetes cluster {} is no longer eligible for alert reconciliation", + kubernetesCluster); + continue; + } + try { + long jobId = scheduleAlertClusterReconciliation(kubernetesCluster); + logger.info("Scheduled Kubernetes cluster {} alert reconciliation as job {}", + kubernetesCluster, jobId); + } catch (RuntimeException e) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw e; + } } else if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkACLItemMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkACLItemMapVO.java new file mode 100644 index 000000000000..08f2e6778655 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkACLItemMapVO.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import java.util.Date; + +import org.apache.cloudstack.api.InternalIdentity; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "kubernetes_cluster_network_acl_item_map") +public class KubernetesClusterNetworkACLItemMapVO implements InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "cluster_id") + private long clusterId; + + @Column(name = "network_acl_item_id") + private long networkAclItemId; + + @Column(name = "logical_role") + private String logicalRole; + + @Enumerated(EnumType.STRING) + @Column(name = "lifecycle_state") + private KubernetesClusterNetworkRuleLifecycleState lifecycleState; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = "updated") + private Date updated; + + public KubernetesClusterNetworkACLItemMapVO() { + } + + public KubernetesClusterNetworkACLItemMapVO(long clusterId, long networkAclItemId, String logicalRole, + KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + this.clusterId = clusterId; + this.networkAclItemId = networkAclItemId; + this.logicalRole = logicalRole; + this.lifecycleState = lifecycleState; + this.updated = new Date(); + } + + @Override + public long getId() { + return id; + } + + public long getNetworkAclItemId() { + return networkAclItemId; + } + + public long getClusterId() { + return clusterId; + } + + public String getLogicalRole() { + return logicalRole; + } + + public KubernetesClusterNetworkRuleLifecycleState getLifecycleState() { + return lifecycleState; + } + + public void setLifecycleState(KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + this.lifecycleState = lifecycleState; + this.updated = new Date(); + } + + public Date getCreated() { + return created; + } + + public Date getUpdated() { + return updated; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpec.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpec.java new file mode 100644 index 000000000000..915f370eb359 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpec.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.exception.InvalidParameterValueException; + +public class KubernetesClusterNetworkRuleAdoptionSpec { + + private static final String RESOURCE_TYPE_KEY = "resourcetype"; + private static final String ROLE_KEY = "role"; + private static final String RESOURCE_ID_KEY = "resourceid"; + private static final String VIRTUAL_MACHINE_ID_KEY = "virtualmachineid"; + private static final Set ALLOWED_KEYS = Set.of(RESOURCE_TYPE_KEY, ROLE_KEY, RESOURCE_ID_KEY, + VIRTUAL_MACHINE_ID_KEY); + + private final KubernetesClusterNetworkRuleRole.ResourceType resourceType; + private final KubernetesClusterNetworkRuleRole role; + private final String resourceUuid; + private final String virtualMachineUuid; + + public KubernetesClusterNetworkRuleAdoptionSpec(KubernetesClusterNetworkRuleRole.ResourceType resourceType, + KubernetesClusterNetworkRuleRole role, String resourceUuid, String virtualMachineUuid) { + this.resourceType = resourceType; + this.role = role; + this.resourceUuid = resourceUuid; + this.virtualMachineUuid = virtualMachineUuid; + } + + public KubernetesClusterNetworkRuleRole.ResourceType getResourceType() { + return resourceType; + } + + public KubernetesClusterNetworkRuleRole getRole() { + return role; + } + + public String getResourceUuid() { + return resourceUuid; + } + + public String getVirtualMachineUuid() { + return virtualMachineUuid; + } + + public String getExternalIdentity() { + return role.name() + ":" + StringUtils.defaultString(virtualMachineUuid); + } + + public static List parse(Map> declarations) { + if (MapUtils.isEmpty(declarations)) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + Set identities = new HashSet<>(); + for (Map.Entry> entry : declarations.entrySet()) { + Map declaration = entry.getValue(); + if (MapUtils.isEmpty(declaration)) { + throw new InvalidParameterValueException(String.format("Network-rule declaration %s is empty", entry.getKey())); + } + Set keys = new HashSet<>(); + declaration.keySet().forEach(key -> { + if (key == null) { + throw new InvalidParameterValueException(String.format( + "Network-rule declaration %s contains a null field name", entry.getKey())); + } + String canonicalKey = key.toLowerCase(Locale.ROOT); + if (!keys.add(canonicalKey)) { + throw new InvalidParameterValueException(String.format( + "Network-rule declaration %s repeats field %s", entry.getKey(), key)); + } + }); + if (!ALLOWED_KEYS.containsAll(keys)) { + keys.removeAll(ALLOWED_KEYS); + throw new InvalidParameterValueException(String.format("Network-rule declaration %s has unsupported fields %s", entry.getKey(), keys)); + } + KubernetesClusterNetworkRuleRole role = parseEnum(KubernetesClusterNetworkRuleRole.class, + value(declaration, ROLE_KEY), "role", entry.getKey()); + KubernetesClusterNetworkRuleRole.ResourceType resourceType = parseEnum(KubernetesClusterNetworkRuleRole.ResourceType.class, + value(declaration, RESOURCE_TYPE_KEY), "resource type", entry.getKey()); + if (!role.getResourceType().equals(resourceType)) { + throw new InvalidParameterValueException(String.format("Role %s requires resource type %s", role, role.getResourceType())); + } + String resourceUuid = StringUtils.trimToNull(value(declaration, RESOURCE_ID_KEY)); + String vmUuid = StringUtils.trimToNull(value(declaration, VIRTUAL_MACHINE_ID_KEY)); + if (resourceUuid == null) { + throw new InvalidParameterValueException(String.format("Network-rule declaration %s is missing resourceId", entry.getKey())); + } + if (role.isVmScoped() != (vmUuid != null)) { + throw new InvalidParameterValueException(String.format("Role %s %s virtualMachineId", role, + role.isVmScoped() ? "requires" : "does not accept")); + } + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec(resourceType, role, + resourceUuid, vmUuid); + if (!identities.add(spec.getExternalIdentity())) { + throw new InvalidParameterValueException(String.format("Duplicate network-rule declaration for %s", spec.getExternalIdentity())); + } + result.add(spec); + } + return result; + } + + private static String value(Map declaration, String key) { + return declaration.entrySet().stream() + .filter(entry -> key.equalsIgnoreCase(entry.getKey())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + } + + private static > T parseEnum(Class type, String value, String field, String declaration) { + if (StringUtils.isBlank(value)) { + throw new InvalidParameterValueException(String.format("Network-rule declaration %s is missing %s", declaration, field)); + } + try { + return Enum.valueOf(type, value.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(String.format("Network-rule declaration %s has invalid %s %s", declaration, field, value)); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleLifecycleState.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleLifecycleState.java new file mode 100644 index 000000000000..88e6de95d945 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleLifecycleState.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +public enum KubernetesClusterNetworkRuleLifecycleState { + PENDING_APPLY, + ACTIVE, + PENDING_DELETE +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleOwnershipState.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleOwnershipState.java new file mode 100644 index 000000000000..8591679f3999 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleOwnershipState.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +public enum KubernetesClusterNetworkRuleOwnershipState { + MANAGED, + LEGACY_UNMANAGED +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleRole.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleRole.java new file mode 100644 index 000000000000..6c6fd8ed3e04 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleRole.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.util.Arrays; + +import com.cloud.utils.exception.CloudRuntimeException; + +public enum KubernetesClusterNetworkRuleRole { + API_FIREWALL(ResourceType.FIREWALL, false), + SSH_FIREWALL(ResourceType.FIREWALL, false), + ETCD_SSH_FIREWALL(ResourceType.FIREWALL, true), + EXTERNAL_SSH_FIREWALL(ResourceType.FIREWALL, true), + API_PORT_FORWARD(ResourceType.PORT_FORWARDING, false), + SSH_PORT_FORWARD(ResourceType.PORT_FORWARDING, true), + API_LOAD_BALANCER(ResourceType.LOAD_BALANCER, false), + API_ACL(ResourceType.NETWORK_ACL_ITEM, false), + SSH_ACL(ResourceType.NETWORK_ACL_ITEM, false), + ETCD_CLIENT_ACL(ResourceType.NETWORK_ACL_ITEM, false); + + public enum ResourceType { + FIREWALL, + PORT_FORWARDING, + LOAD_BALANCER, + NETWORK_ACL_ITEM + } + + private static final String DYNAMIC_SEPARATOR = ":"; + + private final ResourceType resourceType; + private final boolean vmScoped; + + KubernetesClusterNetworkRuleRole(ResourceType resourceType, boolean vmScoped) { + this.resourceType = resourceType; + this.vmScoped = vmScoped; + } + + public ResourceType getResourceType() { + return resourceType; + } + + public boolean isVmScoped() { + return vmScoped; + } + + public String toLogicalRole(Long vmId) { + if (!vmScoped) { + if (vmId != null) { + throw new CloudRuntimeException(String.format("Role %s is not scoped to a virtual machine", name())); + } + return name(); + } + if (vmId == null || vmId <= 0) { + throw new CloudRuntimeException(String.format("Role %s requires a virtual machine", name())); + } + return name() + DYNAMIC_SEPARATOR + vmId; + } + + public static KubernetesClusterNetworkRuleRole fromLogicalRole(String logicalRole) { + if (logicalRole == null) { + throw new CloudRuntimeException("Kubernetes cluster network-rule role is missing"); + } + return Arrays.stream(values()) + .filter(role -> role.vmScoped ? logicalRole.startsWith(role.name() + DYNAMIC_SEPARATOR) : logicalRole.equals(role.name())) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException(String.format("Unsupported Kubernetes cluster network-rule role %s", logicalRole))); + } + + public Long getVmId(String logicalRole) { + if (!vmScoped) { + if (!name().equals(logicalRole)) { + throw new CloudRuntimeException(String.format("Invalid logical role %s for role %s", logicalRole, name())); + } + return null; + } + String prefix = name() + DYNAMIC_SEPARATOR; + if (!logicalRole.startsWith(prefix)) { + throw new CloudRuntimeException(String.format("Invalid logical role %s for role %s", logicalRole, name())); + } + try { + Long vmId = Long.valueOf(logicalRole.substring(prefix.length())); + if (vmId <= 0) { + throw new NumberFormatException("VM ID must be positive"); + } + return vmId; + } catch (NumberFormatException e) { + throw new CloudRuntimeException(String.format("Invalid virtual machine identifier in logical role %s", logicalRole), e); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..b2f6a4348047 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -29,11 +29,13 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ReconcileKubernetesClusterNetworkRulesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterAffinityGroupCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.cluster.AdoptKubernetesClusterNetworkRulesCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -170,6 +172,10 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws CloudRuntimeException; + boolean reconcileKubernetesClusterNetworkRules(ReconcileKubernetesClusterNetworkRulesCmd cmd) throws CloudRuntimeException; + + boolean adoptKubernetesClusterNetworkRules(AdoptKubernetesClusterNetworkRulesCmd cmd) throws CloudRuntimeException; + boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException; boolean updateKubernetesClusterAffinityGroups(UpdateKubernetesClusterAffinityGroupCmd cmd) throws CloudRuntimeException; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 7dfd0043e320..f358a7fc5b06 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -23,6 +23,8 @@ import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.GeneratedValue; @@ -118,6 +120,10 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "cluster_type") private ClusterType clusterType; + @Enumerated(EnumType.STRING) + @Column(name = "network_rule_ownership_state") + private KubernetesClusterNetworkRuleOwnershipState networkRuleOwnershipState; + @Column(name = "control_node_service_offering_id") private Long controlNodeServiceOfferingId; @@ -392,6 +398,14 @@ public void setClusterType(ClusterType clusterType) { this.clusterType = clusterType; } + public KubernetesClusterNetworkRuleOwnershipState getNetworkRuleOwnershipState() { + return networkRuleOwnershipState; + } + + public void setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState networkRuleOwnershipState) { + this.networkRuleOwnershipState = networkRuleOwnershipState; + } + public boolean isCsiEnabled() { return csiEnabled; } @@ -428,6 +442,9 @@ public KubernetesClusterVO(String name, String description, long zoneId, Long ku } this.endpoint = endpoint; this.clusterType = clusterType; + this.networkRuleOwnershipState = ClusterType.CloudManaged.equals(clusterType) + ? KubernetesClusterNetworkRuleOwnershipState.MANAGED + : KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED; this.checkForGc = false; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 4fcf6fa7686d..89c073898f24 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -45,12 +44,14 @@ import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.offering.ServiceOffering; import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.firewall.FirewallService; import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -91,12 +92,19 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleLifecycleState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNetworkACLItemMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; @@ -127,6 +135,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; @@ -152,6 +161,17 @@ public class KubernetesClusterActionWorker { public static final int ETCD_NODE_PEER_COMM_PORT = 2380; public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT; + protected static final String API_FIREWALL_ROLE = KubernetesClusterNetworkRuleRole.API_FIREWALL.name(); + protected static final String SSH_FIREWALL_ROLE = KubernetesClusterNetworkRuleRole.SSH_FIREWALL.name(); + protected static final String API_PORT_FORWARD_ROLE = KubernetesClusterNetworkRuleRole.API_PORT_FORWARD.name(); + protected static final String API_LOAD_BALANCER_ROLE = KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER.name(); + protected static final String API_ACL_ROLE = KubernetesClusterNetworkRuleRole.API_ACL.name(); + protected static final String SSH_ACL_ROLE = KubernetesClusterNetworkRuleRole.SSH_ACL.name(); + protected static final String ETCD_CLIENT_ACL_ROLE = KubernetesClusterNetworkRuleRole.ETCD_CLIENT_ACL.name(); + protected static final String SSH_PORT_FORWARD_ROLE_PREFIX = KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.name() + ":"; + protected static final String ETCD_SSH_FIREWALL_ROLE_PREFIX = KubernetesClusterNetworkRuleRole.ETCD_SSH_FIREWALL.name() + ":"; + protected static final String EXTERNAL_SSH_FIREWALL_ROLE_PREFIX = KubernetesClusterNetworkRuleRole.EXTERNAL_SSH_FIREWALL.name() + ":"; + public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; @@ -217,6 +237,8 @@ public class KubernetesClusterActionWorker { @Inject protected FirewallService firewallService; @Inject + protected FirewallRulesDao firewallRulesDao; + @Inject private NicDao nicDao; @Inject protected AffinityGroupDao affinityGroupDao; @@ -228,6 +250,8 @@ public class KubernetesClusterActionWorker { protected KubernetesClusterDao kubernetesClusterDao; protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + protected KubernetesClusterFirewallRuleMapDao kubernetesClusterFirewallRuleMapDao; + protected KubernetesClusterNetworkACLItemMapDao kubernetesClusterNetworkACLItemMapDao; protected KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; @@ -238,6 +262,147 @@ public class KubernetesClusterActionWorker { protected VirtualMachineTemplate workerNodeTemplate; protected VirtualMachineTemplate etcdTemplate; protected File sshKeyFile; + + protected boolean hasExactSourceCidrs(FirewallRuleVO rule, String... expectedCidrs) { + firewallRulesDao.loadSourceCidrs(rule); + List sourceCidrs = rule.getSourceCidrList(); + return sourceCidrs != null && sourceCidrs.size() == expectedCidrs.length + && new HashSet<>(sourceCidrs).equals(new HashSet<>(Arrays.asList(expectedCidrs))); + } + + protected KubernetesClusterVO lockClusterForNetworkRuleMutation() { + KubernetesClusterVO cluster = kubernetesClusterDao.lockRow(kubernetesCluster.getId(), true); + if (cluster == null) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s no longer exists", kubernetesCluster.getName())); + } + if (cluster.getNetworkRuleOwnershipState() != KubernetesClusterNetworkRuleOwnershipState.MANAGED) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s has network-rule ownership state %s; explicit validated ownership migration is required", + cluster.getName(), cluster.getNetworkRuleOwnershipState())); + } + return cluster; + } + + protected KubernetesClusterFirewallRuleMapVO recordManagedFirewallRule(long ruleId, String logicalRole) { + KubernetesClusterFirewallRuleMapVO existing = kubernetesClusterFirewallRuleMapDao.findByFirewallRuleId(ruleId); + if (existing != null) { + if (existing.getClusterId() != kubernetesCluster.getId() || !Objects.equals(existing.getLogicalRole(), logicalRole)) { + throw new CloudRuntimeException(String.format("Network rule %d is already owned by another Kubernetes cluster", ruleId)); + } + return existing; + } + existing = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole(kubernetesCluster.getId(), logicalRole); + if (existing != null) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s already owns network rule %d for role %s", + kubernetesCluster.getName(), existing.getFirewallRuleId(), logicalRole)); + } + KubernetesClusterFirewallRuleMapVO persisted = kubernetesClusterFirewallRuleMapDao.persist( + new KubernetesClusterFirewallRuleMapVO(kubernetesCluster.getId(), ruleId, logicalRole, + KubernetesClusterNetworkRuleLifecycleState.PENDING_APPLY)); + if (persisted == null) { + throw new CloudRuntimeException(String.format( + "Failed to record ownership for Kubernetes cluster network rule %d", ruleId)); + } + return persisted; + } + + protected KubernetesClusterNetworkACLItemMapVO recordManagedNetworkAclItem(long itemId, String logicalRole) { + KubernetesClusterNetworkACLItemMapVO existing = kubernetesClusterNetworkACLItemMapDao.findByNetworkAclItemId(itemId); + if (existing != null) { + if (existing.getClusterId() != kubernetesCluster.getId() || !Objects.equals(existing.getLogicalRole(), logicalRole)) { + throw new CloudRuntimeException(String.format("Network ACL item %d is already owned by another Kubernetes cluster", itemId)); + } + return existing; + } + existing = kubernetesClusterNetworkACLItemMapDao.findByClusterIdAndLogicalRole(kubernetesCluster.getId(), logicalRole); + if (existing != null) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s already owns network ACL item %d for role %s", + kubernetesCluster.getName(), existing.getNetworkAclItemId(), logicalRole)); + } + KubernetesClusterNetworkACLItemMapVO persisted = kubernetesClusterNetworkACLItemMapDao.persist( + new KubernetesClusterNetworkACLItemMapVO(kubernetesCluster.getId(), itemId, logicalRole, + KubernetesClusterNetworkRuleLifecycleState.PENDING_APPLY)); + if (persisted == null) { + throw new CloudRuntimeException(String.format( + "Failed to record ownership for Kubernetes cluster network ACL item %d", itemId)); + } + return persisted; + } + + protected void activateManagedFirewallRule(long ruleId) { + Transaction.execute((TransactionCallback) status -> { + lockClusterForNetworkRuleMutation(); + KubernetesClusterFirewallRuleMapVO mapping = kubernetesClusterFirewallRuleMapDao.findByFirewallRuleId(ruleId); + if (mapping == null || mapping.getClusterId() != kubernetesCluster.getId()) { + throw new CloudRuntimeException(String.format("Missing ownership for Kubernetes cluster network rule %d", ruleId)); + } + mapping.setLifecycleState(KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + if (!kubernetesClusterFirewallRuleMapDao.update(mapping.getId(), mapping)) { + throw new CloudRuntimeException(String.format("Failed to activate ownership for Kubernetes cluster network rule %d", ruleId)); + } + return null; + }); + } + + protected void activateManagedNetworkAclItem(long itemId) { + Transaction.execute((TransactionCallback) status -> { + lockClusterForNetworkRuleMutation(); + KubernetesClusterNetworkACLItemMapVO mapping = kubernetesClusterNetworkACLItemMapDao.findByNetworkAclItemId(itemId); + if (mapping == null || mapping.getClusterId() != kubernetesCluster.getId()) { + throw new CloudRuntimeException(String.format("Missing ownership for Kubernetes cluster network ACL item %d", itemId)); + } + mapping.setLifecycleState(KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + if (!kubernetesClusterNetworkACLItemMapDao.update(mapping.getId(), mapping)) { + throw new CloudRuntimeException(String.format("Failed to activate ownership for Kubernetes cluster network ACL item %d", itemId)); + } + return null; + }); + } + + protected KubernetesClusterFirewallRuleMapVO markManagedFirewallRulePendingDelete(long ruleId) { + return Transaction.execute((TransactionCallback) status -> { + lockClusterForNetworkRuleMutation(); + KubernetesClusterFirewallRuleMapVO mapping = kubernetesClusterFirewallRuleMapDao.findByFirewallRuleId(ruleId); + if (mapping == null || mapping.getClusterId() != kubernetesCluster.getId()) { + throw new CloudRuntimeException(String.format("Missing ownership for Kubernetes cluster network rule %d", ruleId)); + } + mapping.setLifecycleState(KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE); + if (!kubernetesClusterFirewallRuleMapDao.update(mapping.getId(), mapping)) { + throw new CloudRuntimeException(String.format("Failed to mark ownership pending deletion for Kubernetes cluster network rule %d", ruleId)); + } + return mapping; + }); + } + + protected KubernetesClusterNetworkACLItemMapVO markManagedNetworkAclItemPendingDelete(long itemId) { + return Transaction.execute((TransactionCallback) status -> { + lockClusterForNetworkRuleMutation(); + KubernetesClusterNetworkACLItemMapVO mapping = kubernetesClusterNetworkACLItemMapDao.findByNetworkAclItemId(itemId); + if (mapping == null || mapping.getClusterId() != kubernetesCluster.getId()) { + throw new CloudRuntimeException(String.format("Missing ownership for Kubernetes cluster network ACL item %d", itemId)); + } + mapping.setLifecycleState(KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE); + if (!kubernetesClusterNetworkACLItemMapDao.update(mapping.getId(), mapping)) { + throw new CloudRuntimeException(String.format("Failed to mark ownership pending deletion for Kubernetes cluster network ACL item %d", itemId)); + } + return mapping; + }); + } + + protected void forgetManagedFirewallRule(KubernetesClusterFirewallRuleMapVO mapping) { + KubernetesClusterFirewallRuleMapVO existing = kubernetesClusterFirewallRuleMapDao.findById(mapping.getId()); + if (existing != null && !kubernetesClusterFirewallRuleMapDao.remove(mapping.getId())) { + throw new CloudRuntimeException(String.format("Failed to remove ownership record for Kubernetes cluster network rule %d", + mapping.getFirewallRuleId())); + } + } + + protected void forgetManagedNetworkAclItem(KubernetesClusterNetworkACLItemMapVO mapping) { + KubernetesClusterNetworkACLItemMapVO existing = kubernetesClusterNetworkACLItemMapDao.findById(mapping.getId()); + if (existing != null && !kubernetesClusterNetworkACLItemMapDao.remove(mapping.getId())) { + throw new CloudRuntimeException(String.format("Failed to remove ownership record for Kubernetes cluster network ACL item %d", + mapping.getNetworkAclItemId())); + } + } protected String publicIpAddress; protected int sshPort; @@ -262,6 +427,8 @@ protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluste this.kubernetesCluster = kubernetesCluster; this.kubernetesClusterDao = clusterManager.kubernetesClusterDao; this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao; + this.kubernetesClusterFirewallRuleMapDao = clusterManager.kubernetesClusterFirewallRuleMapDao; + this.kubernetesClusterNetworkACLItemMapDao = clusterManager.kubernetesClusterNetworkACLItemMapDao; this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao; this.kubernetesClusterAffinityGroupMapDao = clusterManager.kubernetesClusterAffinityGroupMapDao; this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao; @@ -485,6 +652,23 @@ protected IpAddress getPublicIp(Network network) throws ManagementServerExceptio return publicIp; } + protected IpAddress getPublicIpForNodeAccess(Network network) throws ManagementServerException { + return manager.isDirectAccess(network) ? null : getPublicIp(network); + } + + protected Pair getNodeIpSshPort(Network network, long nodeId, IpAddress publicIp, + int forwardedPort) throws ManagementServerException { + if (manager.isDirectAccess(network)) { + Nic nic = networkModel.getNicInNetwork(nodeId, network.getId()); + if (nic == null || StringUtils.isBlank(nic.getIPv4Address())) { + throw new ManagementServerException(String.format( + "No IPv4 address was found for node %d on direct-access network %s", nodeId, network.getName())); + } + return new Pair<>(nic.getIPv4Address(), DEFAULT_SSH_PORT); + } + return new Pair<>(publicIp == null ? null : publicIp.getAddress().addr(), forwardedPort); + } + protected IpAddress acquireVpcTierKubernetesPublicIp(Network network, boolean forEtcd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); @@ -681,6 +865,10 @@ protected void updateLoginUserDetails(List clusterVMs) { protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + logger.warn("Cannot transit state on event {} for the Kubernetes cluster with ID: {} as it no longer exists", e, kubernetesClusterId); + return false; + } try { return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); } catch (NoTransitionException nte) { @@ -921,7 +1109,54 @@ protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network n final long domainId = account.getDomainId(); Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); final Ip vmIp = new Ip(vmNic.getIPv4Address()); + String logicalRole = destPort == DEFAULT_SSH_PORT ? SSH_PORT_FORWARD_ROLE_PREFIX + vmId : API_PORT_FORWARD_ROLE; + KubernetesClusterFirewallRuleMapVO existingMapping = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), logicalRole); + if (existingMapping != null) { + PortForwardingRuleVO ownedRule = portForwardingRulesDao.findById(existingMapping.getFirewallRuleId()); + boolean desiredRule = ownedRule != null && !FirewallRule.State.Revoke.equals(ownedRule.getState()) + && Objects.equals(ownedRule.getSourceIpAddressId(), publicIpId) + && Objects.equals(ownedRule.getSourcePortStart(), sourcePort) + && Objects.equals(ownedRule.getSourcePortEnd(), sourcePort) + && Objects.equals(ownedRule.getDestinationPortStart(), destPort) + && Objects.equals(ownedRule.getDestinationPortEnd(), destPort) + && Objects.equals(ownedRule.getVirtualMachineId(), vmId) + && Objects.equals(ownedRule.getNetworkId(), networkId) + && Objects.equals(ownedRule.getAccountId(), accountId) + && Objects.equals(ownedRule.getDomainId(), domainId) + && FirewallRule.Purpose.PortForwarding.equals(ownedRule.getPurpose()) + && NetUtils.TCP_PROTO.equalsIgnoreCase(ownedRule.getProtocol()) + && Objects.equals(ownedRule.getDestinationIpAddress(), vmIp); + if (desiredRule && existingMapping.getLifecycleState() != KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE) { + if (!rulesService.applyPortForwardingRules(publicIpId, account)) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster port forwarding rule", Network.class, networkId); + } + activateManagedFirewallRule(ownedRule.getId()); + return; + } + deleteManagedPortForwardingRule(existingMapping); + } + for (PortForwardingRuleVO existingRule : portForwardingRulesDao.listByIpAndNotRevoked(publicIpId)) { + if (existingRule.getSourcePortStart() > sourcePort || existingRule.getSourcePortEnd() < sourcePort) { + continue; + } + boolean desiredRule = Objects.equals(existingRule.getSourcePortStart(), sourcePort) + && Objects.equals(existingRule.getSourcePortEnd(), sourcePort) + && Objects.equals(existingRule.getDestinationPortStart(), destPort) + && Objects.equals(existingRule.getDestinationPortEnd(), destPort) + && Objects.equals(existingRule.getVirtualMachineId(), vmId) + && Objects.equals(existingRule.getNetworkId(), networkId) + && Objects.equals(existingRule.getAccountId(), accountId) + && NetUtils.TCP_PROTO.equalsIgnoreCase(existingRule.getProtocol()) + && Objects.equals(existingRule.getDestinationIpAddress(), vmIp); + if (desiredRule) { + throw new NetworkRuleConflictException(String.format("Matching port forwarding rule %d is not owned by Kubernetes cluster %s", + existingRule.getId(), kubernetesCluster.getName())); + } + throw new NetworkRuleConflictException(String.format("Public port %d is already used by another port forwarding rule", sourcePort)); + } PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException) status -> { + lockClusterForNetworkRuleMutation(); PortForwardingRuleVO newRule = new PortForwardingRuleVO(null, publicIpId, sourcePort, sourcePort, @@ -931,14 +1166,33 @@ protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network n newRule.setDisplay(true); newRule.setState(FirewallRule.State.Add); newRule = portForwardingRulesDao.persist(newRule); + recordManagedFirewallRule(newRule.getId(), logicalRole); return newRule; }); - rulesService.applyPortForwardingRules(publicIp.getId(), account); + if (!rulesService.applyPortForwardingRules(publicIp.getId(), account)) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster port forwarding rule", Network.class, networkId); + } + activateManagedFirewallRule(pfRule.getId()); if (logger.isInfoEnabled()) { logger.info(String.format("Provisioned SSH port forwarding rule: %s from port %d to %d on %s to the VM IP : %s in Kubernetes cluster : %s", pfRule.getUuid(), sourcePort, destPort, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName())); } } + protected void deleteManagedPortForwardingRule(KubernetesClusterFirewallRuleMapVO mapping) throws ResourceUnavailableException { + markManagedFirewallRulePendingDelete(mapping.getFirewallRuleId()); + FirewallRuleVO rule = firewallRulesDao.findById(mapping.getFirewallRuleId()); + if (rule != null && !rulesService.revokePortForwardingRule(rule.getId(), true)) { + throw new ResourceUnavailableException("Failed to remove the Kubernetes cluster port forwarding rule", Network.class, + kubernetesCluster.getNetworkId()); + } + FirewallRuleVO removedRule = firewallRulesDao.findByIdIncludingRemoved(mapping.getFirewallRuleId()); + if (removedRule != null && removedRule.getRemoved() == null) { + throw new ResourceUnavailableException("Kubernetes cluster port forwarding rule removal was not confirmed", Network.class, + kubernetesCluster.getNetworkId()); + } + forgetManagedFirewallRule(mapping); + } + public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException { String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; @@ -1029,7 +1283,7 @@ public Map addFirewallRulesForNodes(IpAddress publicIp, int size) List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); int endPort = (CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVmList.size() - externalNodes.size() - kubernetesCluster.getEtcdNodeCount().intValue() - 1); - provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, SSH_FIREWALL_ROLE); if (logger.isInfoEnabled()) { logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName())); } @@ -1038,15 +1292,16 @@ public Map addFirewallRulesForNodes(IpAddress publicIp, int size) externalNodes.forEach(externalNode -> { int port = endPort + additionalNodes.get(); try { - provisionFirewallRules(publicIp, owner, port, port); + provisionFirewallRules(publicIp, owner, port, port, + EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + externalNode.getVmId()); vmIdPortMap.put(externalNode.getVmId(), port); - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new CloudRuntimeException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } additionalNodes.addAndGet(1); }); } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } finally { CallContext.unregister(); @@ -1054,36 +1309,77 @@ public Map addFirewallRulesForNodes(IpAddress publicIp, int size) return vmIdPortMap; } - protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, - IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { - List sourceCidrList = new ArrayList(); - sourceCidrList.add("0.0.0.0/0"); - - CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); - rule = ComponentContext.inject(rule); - - Field addressField = rule.getClass().getDeclaredField("ipAddressId"); - addressField.setAccessible(true); - addressField.set(rule, publicIp.getId()); - - Field protocolField = rule.getClass().getDeclaredField("protocol"); - protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); - - Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); - startPortField.setAccessible(true); - startPortField.set(rule, startPort); - - Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); - endPortField.setAccessible(true); - endPortField.set(rule, endPort); + protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort, String logicalRole) + throws ResourceUnavailableException, NetworkRuleConflictException { + List existingRules = firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(publicIp.getId(), startPort, endPort, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall); + KubernetesClusterFirewallRuleMapVO existingMapping = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), logicalRole); + if (existingMapping != null) { + FirewallRuleVO ownedRule = firewallRulesDao.findById(existingMapping.getFirewallRuleId()); + boolean desiredRule = ownedRule != null && !FirewallRule.State.Revoke.equals(ownedRule.getState()) + && Objects.equals(ownedRule.getSourceIpAddressId(), publicIp.getId()) + && Objects.equals(ownedRule.getSourcePortStart(), startPort) + && Objects.equals(ownedRule.getSourcePortEnd(), endPort) + && Objects.equals(ownedRule.getNetworkId(), kubernetesCluster.getNetworkId()) + && Objects.equals(ownedRule.getAccountId(), account.getId()) + && Objects.equals(ownedRule.getDomainId(), account.getDomainId()) + && FirewallRule.Purpose.Firewall.equals(ownedRule.getPurpose()) + && FirewallRule.TrafficType.Ingress.equals(ownedRule.getTrafficType()) + && NetUtils.TCP_PROTO.equalsIgnoreCase(ownedRule.getProtocol()) + && hasExactSourceCidrs(ownedRule, NetUtils.ALL_IP4_CIDRS); + if (desiredRule && existingMapping.getLifecycleState() != KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE) { + if (!firewallService.applyIngressFwRules(publicIp.getId(), account)) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster firewall rule", Network.class, + kubernetesCluster.getNetworkId()); + } + activateManagedFirewallRule(ownedRule.getId()); + return; + } + deleteManagedFirewallRule(existingMapping); + } + for (FirewallRuleVO existingRule : existingRules) { + firewallRulesDao.loadSourceCidrs(existingRule); + if (existingRule.getSourceCidrList() != null && existingRule.getSourceCidrList().contains(NetUtils.ALL_IP4_CIDRS)) { + throw new NetworkRuleConflictException(String.format("Matching firewall rule %d is not owned by Kubernetes cluster %s", + existingRule.getId(), kubernetesCluster.getName())); + } + } - Field cidrField = rule.getClass().getDeclaredField("cidrlist"); - cidrField.setAccessible(true); - cidrField.set(rule, sourceCidrList); + CreateFirewallRuleCmd firewallRule = new CreateFirewallRuleCmd(); + firewallRule = ComponentContext.inject(firewallRule); + firewallRule.setIpAddressId(publicIp.getId()); + firewallRule.setProtocol(NetUtils.TCP_PROTO); + firewallRule.setPublicStartPort(startPort); + firewallRule.setPublicEndPort(endPort); + firewallRule.setSourceCidrList(List.of(NetUtils.ALL_IP4_CIDRS)); + CreateFirewallRuleCmd finalFirewallRule = firewallRule; + FirewallRule createdRule = Transaction.execute((TransactionCallbackWithException) status -> { + lockClusterForNetworkRuleMutation(); + FirewallRule rule = firewallService.createIngressFirewallRule(finalFirewallRule); + recordManagedFirewallRule(rule.getId(), logicalRole); + return rule; + }); + if (!firewallService.applyIngressFwRules(publicIp.getId(), account)) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster firewall rule", Network.class, + kubernetesCluster.getNetworkId()); + } + activateManagedFirewallRule(createdRule.getId()); + } - firewallService.createIngressFirewallRule(rule); - firewallService.applyIngressFwRules(publicIp.getId(), account); + protected void deleteManagedFirewallRule(KubernetesClusterFirewallRuleMapVO mapping) throws ResourceUnavailableException { + markManagedFirewallRulePendingDelete(mapping.getFirewallRuleId()); + FirewallRuleVO rule = firewallRulesDao.findById(mapping.getFirewallRuleId()); + if (rule != null && !firewallService.revokeIngressFwRule(rule.getId(), true)) { + throw new ResourceUnavailableException("Failed to remove the Kubernetes cluster firewall rule", Network.class, + kubernetesCluster.getNetworkId()); + } + FirewallRuleVO removedRule = firewallRulesDao.findByIdIncludingRemoved(mapping.getFirewallRuleId()); + if (removedRule != null && removedRule.getRemoved() == null) { + throw new ResourceUnavailableException("Kubernetes cluster firewall rule removal was not confirmed", Network.class, + kubernetesCluster.getNetworkId()); + } + forgetManagedFirewallRule(mapping); } protected NicVO getVirtualRouterNicOnKubernetesClusterNetwork(KubernetesCluster kubernetesCluster) { @@ -1103,7 +1399,9 @@ protected Map getVmPortMap() { List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); Map vmIdPortMap = new HashMap<>(); - int defaultNodesCount = clusterVmList.size() - externalNodes.size(); + int defaultNodesCount = (int) clusterVmList.stream() + .filter(node -> !node.isExternalNode() && !node.isEtcdNode()) + .count(); AtomicInteger i = new AtomicInteger(0); externalNodes.forEach(node -> { vmIdPortMap.put(node.getVmId(), CLUSTER_NODES_DEFAULT_START_SSH_PORT + defaultNodesCount + i.get()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java index 8b694adf1cca..5c62159c2cb9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -25,15 +25,13 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; -import com.cloud.network.rules.FirewallRuleVO; -import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; import com.cloud.user.Account; import com.cloud.uservm.UserVm; @@ -49,7 +47,6 @@ import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Level; -import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; @@ -60,8 +57,6 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; private long addNodeTimeoutTime; List finalNodeIds = new ArrayList<>(); @@ -80,9 +75,9 @@ public boolean addNodesToCluster(List nodeIds, boolean mountCksIsoOnVr, bo throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); } templateDao.findById(kubernetesCluster.getTemplateId()); - IpAddress publicIp = null; + IpAddress publicIp; try { - publicIp = getPublicIp(network); + publicIp = getPublicIpForNodeAccess(network); } catch (ManagementServerException e) { throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); } @@ -225,13 +220,10 @@ private Pair validateAndSetupNode(Network network, IpAddress p int startSshPortNumber = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - kubernetesCluster.getEtcdNodeCount().intValue(); int sshStartPort = startSshPortNumber + nodeIndex; try { - if (Objects.isNull(network.getVpcId())) { - provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); - } - provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); - boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort); + Pair nodeAccess = prepareNodeAccess(network, publicIp, account, nodeId, sshStartPort); + boolean isCompatible = validateNodeCompatibility(nodeAccess.first(), nodeId, nodeAccess.second()); if (!isCompatible) { - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, nodeId); return new Pair<>(false, nodeIndex); } @@ -245,22 +237,35 @@ private Pair validateAndSetupNode(Network network, IpAddress p idField.set(rebootVMCmd, nodeId); userVmService.rebootVirtualMachine(rebootVMCmd); finalNodeIds.add(nodeId); - } catch (ResourceUnavailableException | NetworkRuleConflictException | NoSuchFieldException | + } catch (ManagementServerException | ResourceUnavailableException | NoSuchFieldException | InsufficientCapacityException | IllegalAccessException e) { logger.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName())); // remove added Firewall and PF rules - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, nodeId); return new Pair<>( false, nodeIndex); } catch (Exception e) { String errMsg = String.format("Unexpected exception while trying to add the external node %s to the Kubernetes cluster %s: %s", nodeId, kubernetesCluster.getName(), e.getMessage()); logger.error(errMsg, e); - revertNetworkRules(network, nodeId, sshStartPort); + revertNetworkRules(network, nodeId); throw new CloudRuntimeException(e); } return new Pair<>(true, ++nodeIndex); } + protected Pair prepareNodeAccess(Network network, IpAddress publicIp, Account account, + long nodeId, int sshStartPort) throws ManagementServerException, ResourceUnavailableException, + NetworkRuleConflictException { + if (!manager.isDirectAccess(network)) { + if (Objects.isNull(network.getVpcId())) { + provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort, + EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + nodeId); + } + provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); + } + return getNodeIpSshPort(network, nodeId, publicIp, sshStartPort); + } + private void updateKubernetesCluster(long clusterId, Ternary additionalNodesDetails, boolean manualUpgrade) { int additionalNodeCount = additionalNodesDetails.first(); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId); @@ -274,14 +279,14 @@ private void updateKubernetesCluster(long clusterId, Ternary result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + Pair result = SshHelper.sshExecute(nodeIp, nodeSshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 10 * 60 * 1000); if (Boolean.FALSE.equals(result.first())) { logger.error(String.format("Node with ID: %s cannot be added as a worker node as it does not have " + @@ -293,14 +298,14 @@ private boolean validateNodeCompatibility(IpAddress publicIp, long nodeId, int n return false; } UserVmVO userVm = userVmDao.findById(nodeId); - cleanupCloudInitSemFolder(userVm, publicIp, pkFile, nodeSshPort); + cleanupCloudInitSemFolder(userVm, nodeIp, pkFile, nodeSshPort); return true; } - private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File pkFile, int nodeSshPort) { + private void cleanupCloudInitSemFolder(UserVm userVm, String nodeIp, File pkFile, int nodeSshPort) { try { String command = String.format("sudo rm -rf /var/lib/cloud/instances/%s/sem/*", userVm.getUuid()); - Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + Pair result = SshHelper.sshExecute(nodeIp, nodeSshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 10 * 60 * 1000); if (Boolean.FALSE.equals(result.first())) { logger.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName())); @@ -310,17 +315,32 @@ private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File p } } - private void revertNetworkRules(Network network, long vmId, int port) { + private void revertNetworkRules(Network network, long vmId) { logger.debug(String.format("Reverting network rules for VM ID %s on network %s", vmId, network.getName())); - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port); - if (Objects.isNull(network.getVpcId())) { - logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); - firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (manager.isDirectAccess(network)) { + return; } - List pfRules = portForwardingRulesDao.listByVm(vmId); - for (PortForwardingRuleVO pfRule : pfRules) { - logger.debug(String.format("Removing port forwarding rule %s", pfRule.getId())); - rulesService.revokePortForwardingRule(pfRule.getId(), true); + KubernetesClusterFirewallRuleMapVO portForwardingOwnership = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), SSH_PORT_FORWARD_ROLE_PREFIX + vmId); + if (portForwardingOwnership != null) { + try { + deleteManagedPortForwardingRule(portForwardingOwnership); + } catch (ResourceUnavailableException e) { + logger.warn("Failed to revert the managed port forwarding rule for VM {} in Kubernetes cluster {}", + vmId, kubernetesCluster.getName(), e); + } + } + if (Objects.isNull(network.getVpcId())) { + KubernetesClusterFirewallRuleMapVO firewallOwnership = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + vmId); + if (firewallOwnership != null) { + try { + deleteManagedFirewallRule(firewallOwnership); + } catch (ResourceUnavailableException e) { + logger.warn("Failed to revert the managed firewall rule for VM {} in Kubernetes cluster {}", + vmId, kubernetesCluster.getName(), e); + } + } } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index dc886117b22e..2aef18a5046a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -17,7 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -42,13 +41,15 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMap; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; -import com.cloud.network.rules.FirewallRule; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; @@ -71,6 +72,10 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod private ASNumberDao asNumberDao; @Inject private BGPService bgpService; + @Inject + protected RemoteAccessVpnDao remoteAccessVpnDao; + @Inject + protected Site2SiteVpnGatewayDao site2SiteVpnGatewayDao; private List clusterVMs; @@ -164,58 +169,22 @@ private void releaseASNumber(Long zoneId, long networkId) { } } - protected void deleteKubernetesClusterIsolatedNetworkRules(Network network, List removedVmIds) throws ManagementServerException { - IpAddress publicIp = getNetworkSourceNatIp(network); - if (publicIp == null) { - throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s", network.getName())); - } - try { - removeLoadBalancingRule(publicIp, network, owner); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to KubernetesCluster load balancing rule for network : %s", network.getName()), e); - } - FirewallRule firewallRule = removeApiFirewallRule(publicIp); - if (firewallRule == null) { - logMessage(Level.WARN, "Firewall rule for API access can't be removed", null); - } - firewallRule = removeSshFirewallRule(publicIp, network.getId()); - if (firewallRule == null) { - logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null); - } - try { - removePortForwardingRules(publicIp, network, owner, removedVmIds); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to KubernetesCluster port forwarding rules for network : %s", network.getName()), e); - } - } - - protected void deleteKubernetesClusterVpcTierRules(Network network, List removedVmIds) throws ManagementServerException { - IpAddress publicIp = getVpcTierKubernetesPublicIp(network); - if (publicIp == null) { + protected void deleteManagedNetworkRulesIfPresent(NetworkVO network, + KubernetesClusterNetworkRuleOwnershipState ownershipState) throws ManagementServerException { + if (!KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(ownershipState)) { return; } - removeVpcTierAclRules(network); - try { - removePortForwardingRules(publicIp, network, owner, removedVmIds); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to KubernetesCluster port forwarding rules for network : %s", network.getName())); - } - } - - private void deleteKubernetesClusterNetworkRules() throws ManagementServerException { - NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { + if (network != null) { + deleteAllManagedNetworkRules(network); return; } - List removedVmIds = new ArrayList<>(); - if (!CollectionUtils.isEmpty(clusterVMs)) { - removedVmIds = clusterVMs.stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); - } - if (network.getVpcId() != null) { - deleteKubernetesClusterVpcTierRules(network, removedVmIds); - return; + boolean hasFirewallMappings = !kubernetesClusterFirewallRuleMapDao.listByClusterId(kubernetesCluster.getId()).isEmpty(); + boolean hasAclMappings = !kubernetesClusterNetworkACLItemMapDao.listByClusterId(kubernetesCluster.getId()).isEmpty(); + if (hasFirewallMappings || hasAclMappings) { + throw new ManagementServerException(String.format( + "Network for Kubernetes cluster %s is missing while managed network-rule ownership records remain", + kubernetesCluster.getName())); } - deleteKubernetesClusterIsolatedNetworkRules(network, removedVmIds); } private void validateClusterVMsDestroyed() { @@ -242,14 +211,8 @@ private void validateClusterVMsDestroyed() { } } - private void checkForRulesToDelete() throws ManagementServerException { - NetworkVO kubernetesClusterNetwork = networkDao.findById(kubernetesCluster.getNetworkId()); - if (kubernetesClusterNetwork != null && !manager.isDirectAccess(kubernetesClusterNetwork)) { - deleteKubernetesClusterNetworkRules(); - } - } - - private void releaseVpcTierPublicIpIfNeeded() throws InsufficientAddressCapacityException { + protected void releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState ownershipState) + throws InsufficientAddressCapacityException { NetworkVO networkVO = networkDao.findById(kubernetesCluster.getNetworkId()); if (networkVO == null || networkVO.getVpcId() == null) { return; @@ -258,7 +221,26 @@ private void releaseVpcTierPublicIpIfNeeded() throws InsufficientAddressCapacity if (address == null) { return; } - networkService.releaseIpAddress(address.getId()); + if (!KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(ownershipState)) { + logger.warn("Retaining public IP {} while deleting legacy Kubernetes cluster {}; ownership of resources on the IP is unknown", + address.getAddress().addr(), kubernetesCluster.getName()); + return; + } + boolean hasDirectIpBinding = address.isSourceNat() || address.isOneToOneNat() + || address.getAssociatedWithVmId() != null || address.getSystem() + || address.isForSystemVms() || address.isForRouter() || address.isPortable(); + boolean hasFirewallRules = CollectionUtils.isNotEmpty(firewallRulesDao.listByIpAndNotRevoked(address.getId())); + boolean hasRemoteAccessVpn = remoteAccessVpnDao.findByPublicIpAddress(address.getId()) != null; + boolean hasSiteToSiteVpnGateway = site2SiteVpnGatewayDao.findByPublicIpAddress(address.getId()) != null; + if (hasDirectIpBinding || hasFirewallRules || hasRemoteAccessVpn || hasSiteToSiteVpnGateway) { + logger.warn("Retaining public IP {} after deleting Kubernetes cluster {} because resources not owned by the cluster remain on the IP", + address.getAddress().addr(), kubernetesCluster.getName()); + return; + } + if (!networkService.releaseIpAddress(address.getId())) { + throw new InsufficientAddressCapacityException(String.format("Failed to release public IP address %s for Kubernetes cluster %s", + address.getAddress().addr(), kubernetesCluster.getName()), Network.class, networkVO.getId()); + } kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID); } @@ -275,6 +257,13 @@ public boolean destroy() throws CloudRuntimeException { if (clusterDetails != null) { cleanupNetwork = Boolean.parseBoolean(clusterDetails.getValue()); } + NetworkVO clusterNetwork = networkDao.findById(kubernetesCluster.getNetworkId()); + boolean directAccess = clusterNetwork != null && manager.isDirectAccess(clusterNetwork); + KubernetesClusterVO persistedCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (persistedCluster == null) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s no longer exists", kubernetesCluster.getName())); + } + boolean legacyNetworkRules = persistedCluster.getNetworkRuleOwnershipState() != KubernetesClusterNetworkRuleOwnershipState.MANAGED; if (cleanupNetwork) { // if network has additional VM, cannot proceed with cluster destroy NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); List externalNodes = clusterVMs.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); @@ -313,25 +302,31 @@ public boolean destroy() throws CloudRuntimeException { if (vmsDestroyed) { if (cleanupNetwork) { validateClusterVMsDestroyed(); + } + try { + deleteManagedNetworkRulesIfPresent(clusterNetwork, persistedCluster.getNetworkRuleOwnershipState()); + } catch (ManagementServerException | CloudRuntimeException e) { + String msg = String.format("Failed to remove managed network rules of Kubernetes cluster: %s", kubernetesCluster); + logger.warn(msg, e); + updateKubernetesClusterEntryForGC(); + throw new CloudRuntimeException(msg, e); + } + if (cleanupNetwork) { try { destroyKubernetesClusterNetwork(); - } catch (ManagementServerException e) { + } catch (ManagementServerException | CloudRuntimeException e) { String msg = String.format("Failed to destroy network of Kubernetes cluster: %s cleanup", kubernetesCluster); logger.warn(msg, e); updateKubernetesClusterEntryForGC(); throw new CloudRuntimeException(msg, e); } } else { - try { - checkForRulesToDelete(); - } catch (ManagementServerException e) { - String msg = String.format("Failed to remove network rules of Kubernetes cluster: %s", kubernetesCluster); - logger.warn(msg, e); - updateKubernetesClusterEntryForGC(); - throw new CloudRuntimeException(msg, e); + if (legacyNetworkRules && !directAccess) { + logger.warn("Leaving unowned legacy network rules in retained network {} while deleting Kubernetes cluster {}; " + + "only explicitly adopted rules can be removed safely", clusterNetwork, kubernetesCluster); } try { - releaseVpcTierPublicIpIfNeeded(); + releaseVpcTierPublicIpIfNeeded(persistedCluster.getNetworkRuleOwnershipState()); } catch (InsufficientAddressCapacityException e) { String msg = String.format("Failed to release public IP for VPC tier used by Kubernetes cluster: %s", kubernetesCluster); logger.warn(msg, e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidator.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidator.java new file mode 100644 index 000000000000..7f16f3a8d435 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidator.java @@ -0,0 +1,813 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleAdoptionSpec; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleLifecycleState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole.ResourceType; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.LoadBalancerVMMapVO; +import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.vpc.NetworkACL; +import com.cloud.network.vpc.NetworkACLItem; +import com.cloud.network.vpc.NetworkACLItemVO; +import com.cloud.network.vpc.NetworkACLVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.Nic; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; + +final class KubernetesClusterNetworkRuleOwnershipValidator { + + private static final Set ADOPTABLE_CLUSTER_STATES = Set.of( + KubernetesCluster.State.Running, KubernetesCluster.State.Stopped, + KubernetesCluster.State.Alert, KubernetesCluster.State.Error); + + private final KubernetesClusterResourceModifierActionWorker worker; + + KubernetesClusterNetworkRuleOwnershipValidator(KubernetesClusterResourceModifierActionWorker worker) { + this.worker = worker; + } + + boolean adopt(List specs) { + List requestedSpecs = specs == null + ? Collections.emptyList() : new ArrayList<>(specs); + return Transaction.execute((TransactionCallback) status -> adoptLocked(requestedSpecs)); + } + + Set getExpectedLogicalRoles(Network network, List vmMaps) { + return new HashSet<>(buildExpectedRules(worker.kubernetesCluster, network, vmMaps).keySet()); + } + + private boolean adoptLocked(List specs) { + KubernetesClusterVO cluster = worker.kubernetesClusterDao.lockRow(worker.kubernetesCluster.getId(), true); + if (cluster == null || cluster.getRemoved() != null) { + throw new InvalidParameterValueException("Kubernetes cluster no longer exists"); + } + if (!KubernetesCluster.ClusterType.CloudManaged.equals(cluster.getClusterType())) { + throw new InvalidParameterValueException("Only CloudManaged Kubernetes clusters can adopt network-rule ownership"); + } + if (!ADOPTABLE_CLUSTER_STATES.contains(cluster.getState())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s is in transient state %s", cluster.getName(), cluster.getState())); + } + KubernetesClusterNetworkRuleOwnershipState ownershipState = cluster.getNetworkRuleOwnershipState(); + if (!KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED.equals(ownershipState) + && !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(ownershipState)) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s has unsupported network-rule ownership state %s", cluster.getName(), ownershipState)); + } + + Network network = worker.networkDao.findById(cluster.getNetworkId()); + if (network == null) { + throw new InvalidParameterValueException(String.format( + "Network for Kubernetes cluster %s cannot be found", cluster.getName())); + } + List vmMaps = worker.kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); + Map vmMapsById = validateClusterVmMaps(cluster, vmMaps); + Map expectedRules = buildExpectedRules(cluster, network, vmMaps); + Map requestedRules = resolveRequestedRules(cluster, specs, vmMapsById, expectedRules); + IpAddress publicIp = expectedRules.isEmpty() ? null : getPublicIp(network); + List adoptedRules = new ArrayList<>(requestedRules.values()); + lockReferencedResources(adoptedRules); + validateNoDuplicateResources(adoptedRules); + validateAndAlignDynamicSshRules(cluster, network, publicIp, expectedRules, requestedRules); + for (Map.Entry entry : expectedRules.entrySet()) { + ExpectedRule expected = expectedRules.get(entry.getKey()); + RequestedRule requested = requestedRules.get(entry.getKey()); + if (requested != null) { + validatePresentResource(cluster, network, publicIp, expected, requested); + } else { + validateAbsentResource(cluster, network, publicIp, expected); + } + } + + if (KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(ownershipState)) { + validateIdempotentReplay(cluster, adoptedRules); + return true; + } + if (!worker.kubernetesClusterFirewallRuleMapDao.listByClusterId(cluster.getId()).isEmpty() + || !worker.kubernetesClusterNetworkACLItemMapDao.listByClusterId(cluster.getId()).isEmpty()) { + throw new CloudRuntimeException(String.format( + "Legacy Kubernetes cluster %s already has network-rule ownership records", cluster.getName())); + } + + for (RequestedRule request : adoptedRules) { + String logicalRole = request.expected.logicalRole; + if (ResourceType.NETWORK_ACL_ITEM.equals(request.spec.getResourceType())) { + KubernetesClusterNetworkACLItemMapVO persisted = worker.kubernetesClusterNetworkACLItemMapDao.persist( + new KubernetesClusterNetworkACLItemMapVO(cluster.getId(), request.resourceId, logicalRole, + KubernetesClusterNetworkRuleLifecycleState.ACTIVE)); + if (persisted == null) { + throw new CloudRuntimeException(String.format("Failed to record ownership of network ACL item %s", + request.spec.getResourceUuid())); + } + } else { + KubernetesClusterFirewallRuleMapVO persisted = worker.kubernetesClusterFirewallRuleMapDao.persist( + new KubernetesClusterFirewallRuleMapVO(cluster.getId(), request.resourceId, logicalRole, + KubernetesClusterNetworkRuleLifecycleState.ACTIVE)); + if (persisted == null) { + throw new CloudRuntimeException(String.format("Failed to record ownership of network rule %s", + request.spec.getResourceUuid())); + } + } + } + cluster.setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + if (!worker.kubernetesClusterDao.update(cluster.getId(), cluster)) { + throw new CloudRuntimeException(String.format( + "Failed to update network-rule ownership state for Kubernetes cluster %s", cluster.getName())); + } + return true; + } + + private Map validateClusterVmMaps(KubernetesClusterVO cluster, + List vmMaps) { + if (vmMaps == null || vmMaps.size() != cluster.getTotalNodeCount()) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s has %d VM mappings but expects %d nodes", cluster.getName(), + vmMaps == null ? 0 : vmMaps.size(), cluster.getTotalNodeCount())); + } + Map result = new LinkedHashMap<>(); + long controlNodeCount = 0; + long etcdNodeCount = 0; + long workerNodeCount = 0; + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + int assignedRoles = (vmMap.isControlNode() ? 1 : 0) + (vmMap.isEtcdNode() ? 1 : 0) + + (vmMap.isExternalNode() ? 1 : 0); + if (assignedRoles > 1) { + throw new InvalidParameterValueException(String.format( + "Mapped VM %d for Kubernetes cluster %s has conflicting node roles", + vmMap.getVmId(), cluster.getName())); + } + if (vmMap.isControlNode()) { + controlNodeCount++; + } else if (vmMap.isEtcdNode()) { + etcdNodeCount++; + } else { + workerNodeCount++; + } + if (result.put(vmMap.getVmId(), vmMap) != null) { + throw new CloudRuntimeException(String.format( + "Kubernetes cluster %s contains duplicate VM mapping %d", cluster.getName(), vmMap.getVmId())); + } + VMInstanceVO vm = worker.vmInstanceDao.findById(vmMap.getVmId()); + if (vm == null || VirtualMachine.State.Destroyed.equals(vm.getState()) + || VirtualMachine.State.Expunging.equals(vm.getState())) { + throw new InvalidParameterValueException(String.format( + "Mapped VM %d for Kubernetes cluster %s is unavailable", vmMap.getVmId(), cluster.getName())); + } + if (vm.getAccountId() != cluster.getAccountId() || vm.getDomainId() != cluster.getDomainId()) { + throw new InvalidParameterValueException(String.format( + "Mapped VM %s does not belong to Kubernetes cluster %s", vm.getUuid(), cluster.getName())); + } + } + if (controlNodeCount != cluster.getControlNodeCount() + || etcdNodeCount != cluster.getEtcdNodeCount() + || workerNodeCount != cluster.getNodeCount()) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s VM mapping roles do not match its configured node counts", + cluster.getName())); + } + return result; + } + + private Map buildExpectedRules(KubernetesCluster cluster, Network network, + List vmMaps) { + Map expected = new LinkedHashMap<>(); + if (worker.manager.isDirectAccess(network)) { + return expected; + } + List control = vmMaps.stream().filter(KubernetesClusterVmMapVO::isControlNode) + .collect(Collectors.toList()); + List etcd = vmMaps.stream().filter(KubernetesClusterVmMapVO::isEtcdNode) + .collect(Collectors.toList()); + List external = vmMaps.stream().filter(KubernetesClusterVmMapVO::isExternalNode) + .collect(Collectors.toList()); + List standard = vmMaps.stream() + .filter(vm -> !vm.isEtcdNode() && !vm.isExternalNode()).collect(Collectors.toList()); + if (control.isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s has no control-node mapping", cluster.getName())); + } + + int standardSshEnd = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + standard.size() - 1; + if (network.getVpcId() == null) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.API_FIREWALL, null, + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, null)); + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.SSH_FIREWALL, null, + KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT, standardSshEnd, null)); + for (int i = 0; i < external.size(); i++) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.EXTERNAL_SSH_FIREWALL, + external.get(i), standardSshEnd + i + 1, standardSshEnd + i + 1, null)); + } + int etcdStartPort = com.cloud.kubernetes.cluster.KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + for (int i = 0; i < etcd.size(); i++) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.ETCD_SSH_FIREWALL, + etcd.get(i), etcdStartPort + i, etcdStartPort + i, null)); + } + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER, null, + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, + KubernetesClusterActionWorker.CLUSTER_API_PORT)); + } else { + validateVpcTierAclTopology(cluster, network); + if (!Objects.equals(network.getNetworkACLId(), NetworkACL.DEFAULT_ALLOW)) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.API_ACL, null, + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, null)); + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.SSH_ACL, null, + KubernetesClusterActionWorker.DEFAULT_SSH_PORT, KubernetesClusterActionWorker.DEFAULT_SSH_PORT, null)); + if (!etcd.isEmpty()) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.ETCD_CLIENT_ACL, null, + KubernetesClusterActionWorker.ETCD_NODE_CLIENT_REQUEST_PORT, + KubernetesClusterActionWorker.ETCD_NODE_CLIENT_REQUEST_PORT, null)); + } + } + NetworkOffering offering = worker.networkOfferingDao.findById(network.getNetworkOfferingId()); + if (offering == null) { + throw new InvalidParameterValueException(String.format( + "Network offering for Kubernetes cluster %s cannot be found", cluster.getName())); + } + if (offering.isConserveMode()) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER, null, + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, + KubernetesClusterActionWorker.CLUSTER_API_PORT)); + } else { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.API_PORT_FORWARD, control.get(0), + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, + KubernetesClusterActionWorker.CLUSTER_API_PORT)); + } + } + + int nextStandardPort = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT; + int externalPortBase = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + standard.size(); + int externalIndex = 0; + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + if (vmMap.isEtcdNode()) { + continue; + } + int sourcePort = vmMap.isExternalNode() ? externalPortBase + externalIndex++ : nextStandardPort++; + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, vmMap, + sourcePort, sourcePort, KubernetesClusterActionWorker.DEFAULT_SSH_PORT)); + } + int etcdStartPort = com.cloud.kubernetes.cluster.KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + for (int i = 0; i < etcd.size(); i++) { + putExpected(expected, new ExpectedRule(KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, etcd.get(i), + etcdStartPort + i, etcdStartPort + i, KubernetesClusterActionWorker.DEFAULT_SSH_PORT)); + } + return expected; + } + + private void validateVpcTierAclTopology(KubernetesCluster cluster, Network network) { + Long networkAclId = network.getNetworkACLId(); + if (networkAclId == null) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s VPC tier does not have a network ACL attached", cluster.getName())); + } + if (Objects.equals(networkAclId, NetworkACL.DEFAULT_DENY)) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s VPC tier uses the default-deny ACL and cannot have cluster-owned ACL rules", + cluster.getName())); + } + if (Objects.equals(networkAclId, NetworkACL.DEFAULT_ALLOW)) { + return; + } + NetworkACLVO acl = worker.networkACLDao.findById(networkAclId); + if (acl == null || !Objects.equals(acl.getVpcId(), network.getVpcId())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s VPC tier has an invalid network ACL", cluster.getName())); + } + if (worker.networkDao.listByAclId(networkAclId).stream() + .anyMatch(attached -> attached.getId() != network.getId())) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s VPC tier network ACL is shared by another tier", + cluster.getName())); + } + } + + private void putExpected(Map expected, ExpectedRule rule) { + if (expected.put(rule.logicalRole, rule) != null) { + throw new CloudRuntimeException(String.format("Duplicate expected Kubernetes network-rule role %s", rule.logicalRole)); + } + } + + private Map resolveRequestedRules(KubernetesClusterVO cluster, + List specs, + Map vmMapsById, Map expectedRules) { + Map requested = new LinkedHashMap<>(); + for (KubernetesClusterNetworkRuleAdoptionSpec spec : specs) { + KubernetesClusterVmMapVO vmMap = null; + if (spec.getVirtualMachineUuid() != null) { + VMInstanceVO vm = worker.vmInstanceDao.findByUuid(spec.getVirtualMachineUuid()); + if (vm == null || (vmMap = vmMapsById.get(vm.getId())) == null) { + throw new InvalidParameterValueException(String.format( + "VM %s is not mapped to Kubernetes cluster %s", spec.getVirtualMachineUuid(), cluster.getName())); + } + validateVmScopedRole(spec.getRole(), vmMap); + } + String logicalRole = spec.getRole().toLogicalRole(vmMap == null ? null : vmMap.getVmId()); + ExpectedRule expected = expectedRules.get(logicalRole); + if (expected == null) { + throw new InvalidParameterValueException(String.format( + "Role %s is not expected for Kubernetes cluster %s", logicalRole, cluster.getName())); + } + RequestedRule value = new RequestedRule(spec, expected); + if (requested.put(logicalRole, value) != null) { + throw new InvalidParameterValueException(String.format("Duplicate declaration for role %s", logicalRole)); + } + } + return requested; + } + + private void validateVmScopedRole(KubernetesClusterNetworkRuleRole role, KubernetesClusterVmMapVO vmMap) { + if (KubernetesClusterNetworkRuleRole.ETCD_SSH_FIREWALL.equals(role) && !vmMap.isEtcdNode()) { + throw new InvalidParameterValueException("ETCD_SSH_FIREWALL requires an etcd-node VM"); + } + if (KubernetesClusterNetworkRuleRole.EXTERNAL_SSH_FIREWALL.equals(role) && !vmMap.isExternalNode()) { + throw new InvalidParameterValueException("EXTERNAL_SSH_FIREWALL requires an external-node VM"); + } + } + + private IpAddress getPublicIp(Network network) { + try { + IpAddress publicIp = worker.getPublicIp(network); + if (publicIp.getAllocatedToAccountId() == null + || publicIp.getAllocatedToAccountId() != worker.kubernetesCluster.getAccountId() + || publicIp.getAllocatedInDomainId() == null + || publicIp.getAllocatedInDomainId() != worker.kubernetesCluster.getDomainId() + || !Objects.equals(publicIp.getAssociatedWithNetworkId(), network.getId())) { + throw new InvalidParameterValueException(String.format( + "Public IP %s does not belong to Kubernetes cluster %s network and owner", + publicIp.getUuid(), worker.kubernetesCluster.getName())); + } + return publicIp; + } catch (ManagementServerException e) { + throw new InvalidParameterValueException(e.getMessage()); + } + } + + private void lockReferencedResources(List presentRules) { + List firewallIds = new ArrayList<>(); + List aclItemIds = new ArrayList<>(); + for (RequestedRule requested : presentRules) { + Long id = resolveResourceId(requested.spec); + requested.resourceId = id; + if (ResourceType.NETWORK_ACL_ITEM.equals(requested.spec.getResourceType())) { + aclItemIds.add(id); + } else { + firewallIds.add(id); + } + } + firewallIds.stream().sorted().forEach(id -> { + if (worker.firewallRulesDao.lockRow(id, true) == null) { + throw new InvalidParameterValueException(String.format("Network rule %d no longer exists", id)); + } + }); + aclItemIds.stream().sorted().forEach(id -> { + if (worker.networkACLItemDao.lockRow(id, true) == null) { + throw new InvalidParameterValueException(String.format("Network ACL item %d no longer exists", id)); + } + }); + } + + private Long resolveResourceId(KubernetesClusterNetworkRuleAdoptionSpec spec) { + switch (spec.getResourceType()) { + case FIREWALL: + return requireResource(worker.firewallRulesDao.findByUuid(spec.getResourceUuid()), spec).getId(); + case PORT_FORWARDING: + return requireResource(worker.portForwardingRulesDao.findByUuid(spec.getResourceUuid()), spec).getId(); + case LOAD_BALANCER: + return requireResource(worker.loadBalancerDao.findByUuid(spec.getResourceUuid()), spec).getId(); + case NETWORK_ACL_ITEM: + return requireResource(worker.networkACLItemDao.findByUuid(spec.getResourceUuid()), spec).getId(); + default: + throw new InvalidParameterValueException(String.format("Unsupported resource type %s", spec.getResourceType())); + } + } + + private T requireResource(T resource, KubernetesClusterNetworkRuleAdoptionSpec spec) { + if (resource == null) { + throw new InvalidParameterValueException(String.format("%s resource %s cannot be found", + spec.getResourceType(), spec.getResourceUuid())); + } + return resource; + } + + private void validateNoDuplicateResources(List presentRules) { + Set resourceKeys = new HashSet<>(); + for (RequestedRule request : presentRules) { + String key = request.spec.getResourceType() + ":" + request.resourceId; + if (!resourceKeys.add(key)) { + throw new InvalidParameterValueException(String.format( + "Resource %s is declared for more than one network-rule role", request.spec.getResourceUuid())); + } + } + } + + private void validateAndAlignDynamicSshRules(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + Map expectedRules, Map requestedRules) { + List dynamicSshRules = expectedRules.values().stream() + .filter(this::isDynamicSshPortForwardingRole) + .collect(Collectors.toList()); + if (dynamicSshRules.isEmpty()) { + return; + } + List nonRevokedPortForwardingRules = + worker.portForwardingRulesDao.listByIpAndNotRevoked(publicIp.getId()); + for (ExpectedRule expected : dynamicSshRules) { + List candidates = nonRevokedPortForwardingRules.stream() + .filter(rule -> isSshPortForwardingCandidate(cluster, network, publicIp, expected, rule)) + .collect(Collectors.toList()); + if (candidates.size() > 1) { + throw new InvalidParameterValueException(String.format( + "More than one non-revoked SSH port-forwarding rule targets VM %s for role %s", + expected.vmMap.getVmId(), expected.logicalRole)); + } + RequestedRule requested = requestedRules.get(expected.logicalRole); + if (requested == null) { + if (!candidates.isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Role %s was omitted but non-revoked resource %s targets its mapped VM", + expected.logicalRole, candidates.get(0).getUuid())); + } + continue; + } + if (candidates.isEmpty() || !Objects.equals(candidates.get(0).getId(), requested.resourceId)) { + invalidResource(requested.spec.getResourceUuid(), expected, + "resource must be the only non-revoked SSH port-forwarding rule targeting the mapped VM"); + } + PortForwardingRuleVO rule = candidates.get(0); + Integer sourcePort = rule.getSourcePortStart(); + if (sourcePort == null || sourcePort <= 0 || sourcePort > NetUtils.PORT_RANGE_MAX + || !sourcePort.equals(rule.getSourcePortEnd())) { + invalidResource(rule.getUuid(), expected, "SSH source port must be one valid port"); + } + expected.setSourcePorts(sourcePort, sourcePort); + KubernetesClusterNetworkRuleRole firewallRole = expected.vmMap.isEtcdNode() + ? KubernetesClusterNetworkRuleRole.ETCD_SSH_FIREWALL + : KubernetesClusterNetworkRuleRole.EXTERNAL_SSH_FIREWALL; + ExpectedRule firewall = expectedRules.get(firewallRole.toLogicalRole(expected.vmMap.getVmId())); + if (firewall != null) { + firewall.setSourcePorts(sourcePort, sourcePort); + } + } + for (RequestedRule requested : requestedRules.values()) { + if (!isDynamicSshFirewallRole(requested.expected)) { + continue; + } + String portForwardingRole = KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD + .toLogicalRole(requested.expected.vmMap.getVmId()); + if (!requestedRules.containsKey(portForwardingRole)) { + throw new InvalidParameterValueException(String.format( + "%s for VM %s requires the corresponding SSH_PORT_FORWARD declaration", + requested.expected.role, requested.expected.vmMap.getVmId())); + } + } + } + + private boolean isDynamicSshPortForwardingRole(ExpectedRule expected) { + return KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.equals(expected.role) + && expected.vmMap != null && (expected.vmMap.isEtcdNode() || expected.vmMap.isExternalNode()); + } + + private boolean isDynamicSshFirewallRole(ExpectedRule expected) { + return KubernetesClusterNetworkRuleRole.ETCD_SSH_FIREWALL.equals(expected.role) + || KubernetesClusterNetworkRuleRole.EXTERNAL_SSH_FIREWALL.equals(expected.role); + } + + private boolean isSshPortForwardingCandidate(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected, PortForwardingRuleVO rule) { + return !FirewallRule.State.Revoke.equals(rule.getState()) + && FirewallRule.Purpose.PortForwarding.equals(rule.getPurpose()) + && Objects.equals(rule.getSourceIpAddressId(), publicIp.getId()) + && Objects.equals(rule.getNetworkId(), network.getId()) + && rule.getAccountId() == cluster.getAccountId() + && rule.getDomainId() == cluster.getDomainId() + && NetUtils.TCP_PROTO.equalsIgnoreCase(rule.getProtocol()) + && Objects.equals(rule.getVirtualMachineId(), expected.vmMap.getVmId()) + && Objects.equals(rule.getDestinationPortStart(), KubernetesClusterActionWorker.DEFAULT_SSH_PORT) + && Objects.equals(rule.getDestinationPortEnd(), KubernetesClusterActionWorker.DEFAULT_SSH_PORT); + } + + private void validatePresentResource(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected, RequestedRule request) { + switch (request.spec.getResourceType()) { + case FIREWALL: + validateFirewallRule(cluster, network, publicIp, expected, + requireResource(worker.firewallRulesDao.findById(request.resourceId), request.spec)); + break; + case PORT_FORWARDING: + validatePortForwardingRule(cluster, network, publicIp, expected, + requireResource(worker.portForwardingRulesDao.findById(request.resourceId), request.spec)); + break; + case LOAD_BALANCER: + validateLoadBalancerRule(cluster, network, publicIp, + requireResource(worker.loadBalancerDao.findById(request.resourceId), request.spec)); + break; + case NETWORK_ACL_ITEM: + validateNetworkAclItem(cluster, network, expected, + requireResource(worker.networkACLItemDao.findById(request.resourceId), request.spec)); + break; + default: + throw new InvalidParameterValueException(String.format("Unsupported resource type %s", request.spec.getResourceType())); + } + validateResourceIsUnownedOrExact(cluster, request); + } + + private void validateFirewallRule(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected, FirewallRuleVO rule) { + validateBaseFirewallRule(cluster, network, publicIp, expected, rule, FirewallRule.Purpose.Firewall); + if (!FirewallRule.TrafficType.Ingress.equals(rule.getTrafficType())) { + invalidResource(rule.getUuid(), expected, "traffic type must be Ingress"); + } + if (!worker.hasExactSourceCidrs(rule, NetUtils.ALL_IP4_CIDRS)) { + invalidResource(rule.getUuid(), expected, "source CIDR must be exactly 0.0.0.0/0"); + } + } + + private void validatePortForwardingRule(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected, PortForwardingRuleVO rule) { + validateBaseFirewallRule(cluster, network, publicIp, expected, rule, FirewallRule.Purpose.PortForwarding); + if (!Objects.equals(rule.getDestinationPortStart(), expected.destinationPort) + || !Objects.equals(rule.getDestinationPortEnd(), expected.destinationPort) + || !Objects.equals(rule.getVirtualMachineId(), expected.vmMap.getVmId())) { + invalidResource(rule.getUuid(), expected, "destination port or VM does not match"); + } + Nic nic = worker.networkModel.getNicInNetwork(expected.vmMap.getVmId(), network.getId()); + if (nic == null || !Objects.equals(rule.getDestinationIpAddress(), new Ip(nic.getIPv4Address()))) { + invalidResource(rule.getUuid(), expected, "destination IP is not the mapped VM network address"); + } + } + + private void validateLoadBalancerRule(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + LoadBalancerVO rule) { + ExpectedRule expected = new ExpectedRule(KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER, null, + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT, + KubernetesClusterActionWorker.CLUSTER_API_PORT); + validateBaseFirewallRule(cluster, network, publicIp, expected, rule, FirewallRule.Purpose.LoadBalancing); + if (!Objects.equals(rule.getDefaultPortStart(), expected.destinationPort) + || !Objects.equals(rule.getDefaultPortEnd(), expected.destinationPort) + || !"api-lb".equals(rule.getName()) || !"roundrobin".equalsIgnoreCase(rule.getAlgorithm()) + || !NetUtils.TCP_PROTO.equalsIgnoreCase(rule.getLbProtocol())) { + invalidResource(rule.getUuid(), expected, "load-balancer policy does not match the managed API rule"); + } + Map expectedBackends = new HashMap<>(); + for (KubernetesClusterVmMapVO vmMap : worker.kubernetesClusterVmMapDao.listByClusterId(cluster.getId())) { + if (!vmMap.isControlNode()) { + continue; + } + Nic nic = worker.networkModel.getNicInNetwork(vmMap.getVmId(), network.getId()); + if (nic == null || expectedBackends.put(vmMap.getVmId(), nic.getIPv4Address()) != null) { + invalidResource(rule.getUuid(), expected, "control-node backend set cannot be resolved"); + } + } + Map actualBackends = new HashMap<>(); + for (LoadBalancerVMMapVO mapping : worker.loadBalancerVMMapDao.listByLoadBalancerId(rule.getId(), false)) { + if (mapping.isRevoke() || actualBackends.put(mapping.getInstanceId(), mapping.getInstanceIp()) != null) { + invalidResource(rule.getUuid(), expected, "load-balancer backend set contains duplicate or revoked entries"); + } + } + if (!expectedBackends.equals(actualBackends)) { + invalidResource(rule.getUuid(), expected, "load-balancer backend set does not exactly match control nodes"); + } + } + + private void validateBaseFirewallRule(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected, FirewallRuleVO rule, FirewallRule.Purpose purpose) { + if (!FirewallRule.State.Active.equals(rule.getState()) || !purpose.equals(rule.getPurpose()) + || !Objects.equals(rule.getSourceIpAddressId(), publicIp.getId()) + || !Objects.equals(rule.getNetworkId(), network.getId()) + || rule.getAccountId() != cluster.getAccountId() || rule.getDomainId() != cluster.getDomainId() + || !NetUtils.TCP_PROTO.equalsIgnoreCase(rule.getProtocol()) + || !Objects.equals(rule.getSourcePortStart(), expected.sourcePortStart) + || !Objects.equals(rule.getSourcePortEnd(), expected.sourcePortEnd)) { + invalidResource(rule.getUuid(), expected, "scope, state, protocol, or source ports do not match"); + } + } + + private void validateNetworkAclItem(KubernetesClusterVO cluster, Network network, ExpectedRule expected, + NetworkACLItemVO item) { + Long networkAclId = network.getNetworkACLId(); + if (networkAclId == null || networkAclId <= NetworkACL.DEFAULT_DENY) { + invalidResource(item.getUuid(), expected, "cluster tier does not have a dedicated network ACL"); + } + NetworkACLVO acl = worker.networkACLDao.findById(item.getAclId()); + if (acl == null || item.getAclId() != networkAclId || !Objects.equals(acl.getVpcId(), network.getVpcId()) + || worker.networkDao.listByAclId(networkAclId).stream().anyMatch(attached -> attached.getId() != network.getId())) { + invalidResource(item.getUuid(), expected, "network ACL is not dedicated and attached to the cluster tier"); + } + worker.networkACLItemDao.loadCidrs(item); + if (!NetworkACLItem.State.Active.equals(item.getState()) + || !NetUtils.TCP_PROTO.equalsIgnoreCase(item.getProtocol()) + || !Objects.equals(item.getSourcePortStart(), expected.sourcePortStart) + || !Objects.equals(item.getSourcePortEnd(), expected.sourcePortEnd) + || !NetworkACLItem.TrafficType.Ingress.equals(item.getTrafficType()) + || !NetworkACLItem.Action.Allow.equals(item.getAction()) + || item.getSourceCidrList() == null || item.getSourceCidrList().size() != 2 + || !new HashSet<>(item.getSourceCidrList()).equals(Set.of(NetUtils.ALL_IP4_CIDRS, NetUtils.ALL_IP6_CIDRS))) { + invalidResource(item.getUuid(), expected, "state, protocol, ports, action, traffic type, or CIDRs do not match"); + } + } + + private void validateResourceIsUnownedOrExact(KubernetesClusterVO cluster, RequestedRule request) { + String logicalRole = request.expected.logicalRole; + if (ResourceType.NETWORK_ACL_ITEM.equals(request.spec.getResourceType())) { + KubernetesClusterNetworkACLItemMapVO byResource = worker.kubernetesClusterNetworkACLItemMapDao.findByNetworkAclItemId(request.resourceId); + KubernetesClusterNetworkACLItemMapVO byRole = worker.kubernetesClusterNetworkACLItemMapDao.findByClusterIdAndLogicalRole(cluster.getId(), logicalRole); + validateExistingOwnership(cluster, logicalRole, request.resourceId, byResource == null ? null : byResource.getClusterId(), + byResource == null ? null : byResource.getLogicalRole(), byRole == null ? null : byRole.getNetworkAclItemId()); + } else { + KubernetesClusterFirewallRuleMapVO byResource = worker.kubernetesClusterFirewallRuleMapDao.findByFirewallRuleId(request.resourceId); + KubernetesClusterFirewallRuleMapVO byRole = worker.kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole(cluster.getId(), logicalRole); + validateExistingOwnership(cluster, logicalRole, request.resourceId, byResource == null ? null : byResource.getClusterId(), + byResource == null ? null : byResource.getLogicalRole(), byRole == null ? null : byRole.getFirewallRuleId()); + } + } + + private void validateExistingOwnership(KubernetesClusterVO cluster, String logicalRole, long resourceId, + Long ownerClusterId, String ownedRole, Long roleResourceId) { + if (ownerClusterId != null && (ownerClusterId != cluster.getId() || !logicalRole.equals(ownedRole))) { + throw new InvalidParameterValueException(String.format("Resource for role %s is already owned by another mapping", logicalRole)); + } + if (roleResourceId != null && roleResourceId != resourceId) { + throw new InvalidParameterValueException(String.format("Role %s is already mapped to another resource", logicalRole)); + } + } + + private void validateAbsentResource(KubernetesClusterVO cluster, Network network, IpAddress publicIp, + ExpectedRule expected) { + List conflicts = new ArrayList<>(); + switch (expected.role.getResourceType()) { + case FIREWALL: + for (FirewallRuleVO rule : worker.firewallRulesDao.listByIpAndPurposeAndNotRevoked( + publicIp.getId(), FirewallRule.Purpose.Firewall)) { + if (overlaps(expected, rule)) { + conflicts.add(rule.getUuid()); + } + } + break; + case PORT_FORWARDING: + for (PortForwardingRuleVO rule : worker.portForwardingRulesDao.listByIpAndNotRevoked(publicIp.getId())) { + if (overlaps(expected, rule)) { + conflicts.add(rule.getUuid()); + } + } + break; + case LOAD_BALANCER: + for (LoadBalancerVO rule : worker.loadBalancerDao.listByIpAddress(publicIp.getId())) { + if (!FirewallRule.State.Revoke.equals(rule.getState()) && overlaps(expected, rule)) { + conflicts.add(rule.getUuid()); + } + } + break; + case NETWORK_ACL_ITEM: + if (network.getNetworkACLId() != null) { + for (NetworkACLItemVO item : worker.networkACLItemDao.listByACL(network.getNetworkACLId())) { + if (!NetworkACLItem.State.Revoke.equals(item.getState()) + && NetUtils.TCP_PROTO.equalsIgnoreCase(item.getProtocol()) + && rangesOverlap(expected.sourcePortStart, expected.sourcePortEnd, + item.getSourcePortStart(), item.getSourcePortEnd())) { + conflicts.add(item.getUuid()); + } + } + } + break; + default: + throw new InvalidParameterValueException(String.format("Unsupported resource type %s", expected.role.getResourceType())); + } + if (!conflicts.isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Role %s was omitted but conflicting unowned resources exist: %s", + expected.logicalRole, conflicts)); + } + if (ResourceType.NETWORK_ACL_ITEM.equals(expected.role.getResourceType())) { + if (worker.kubernetesClusterNetworkACLItemMapDao.findByClusterIdAndLogicalRole(cluster.getId(), expected.logicalRole) != null) { + throw new InvalidParameterValueException(String.format("Role %s already has an ownership mapping", expected.logicalRole)); + } + } else if (worker.kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole(cluster.getId(), expected.logicalRole) != null) { + throw new InvalidParameterValueException(String.format("Role %s already has an ownership mapping", expected.logicalRole)); + } + } + + private boolean overlaps(ExpectedRule expected, FirewallRuleVO rule) { + return NetUtils.TCP_PROTO.equalsIgnoreCase(rule.getProtocol()) + && rangesOverlap(expected.sourcePortStart, expected.sourcePortEnd, + rule.getSourcePortStart(), rule.getSourcePortEnd()); + } + + private boolean rangesOverlap(Integer firstStart, Integer firstEnd, Integer secondStart, Integer secondEnd) { + return firstStart != null && firstEnd != null && secondStart != null && secondEnd != null + && firstStart <= secondEnd && secondStart <= firstEnd; + } + + private void validateIdempotentReplay(KubernetesClusterVO cluster, List presentRules) { + Map expectedFirewallMappings = presentRules.stream() + .filter(request -> !ResourceType.NETWORK_ACL_ITEM.equals(request.spec.getResourceType())) + .collect(Collectors.toMap(request -> request.expected.logicalRole, request -> request.resourceId)); + Map actualFirewallMappings = worker.kubernetesClusterFirewallRuleMapDao.listByClusterId(cluster.getId()).stream() + .peek(mapping -> { + if (!KubernetesClusterNetworkRuleLifecycleState.ACTIVE.equals(mapping.getLifecycleState())) { + throw new InvalidParameterValueException(String.format("Role %s is not active", mapping.getLogicalRole())); + } + }) + .collect(Collectors.toMap(KubernetesClusterFirewallRuleMapVO::getLogicalRole, + KubernetesClusterFirewallRuleMapVO::getFirewallRuleId)); + Map expectedAclMappings = presentRules.stream() + .filter(request -> ResourceType.NETWORK_ACL_ITEM.equals(request.spec.getResourceType())) + .collect(Collectors.toMap(request -> request.expected.logicalRole, request -> request.resourceId)); + Map actualAclMappings = worker.kubernetesClusterNetworkACLItemMapDao.listByClusterId(cluster.getId()).stream() + .peek(mapping -> { + if (!KubernetesClusterNetworkRuleLifecycleState.ACTIVE.equals(mapping.getLifecycleState())) { + throw new InvalidParameterValueException(String.format("Role %s is not active", mapping.getLogicalRole())); + } + }) + .collect(Collectors.toMap(KubernetesClusterNetworkACLItemMapVO::getLogicalRole, + KubernetesClusterNetworkACLItemMapVO::getNetworkAclItemId)); + if (!expectedFirewallMappings.equals(actualFirewallMappings) || !expectedAclMappings.equals(actualAclMappings)) { + throw new InvalidParameterValueException(String.format( + "Kubernetes cluster %s is already managed and the supplied manifest is not an exact replay", cluster.getName())); + } + } + + private void invalidResource(String resourceUuid, ExpectedRule expected, String detail) { + throw new InvalidParameterValueException(String.format( + "Resource %s does not match Kubernetes network-rule role %s: %s", + resourceUuid, expected.logicalRole, detail)); + } + + private static final class ExpectedRule { + private final KubernetesClusterNetworkRuleRole role; + private final KubernetesClusterVmMapVO vmMap; + private final String logicalRole; + private Integer sourcePortStart; + private Integer sourcePortEnd; + private final Integer destinationPort; + + private ExpectedRule(KubernetesClusterNetworkRuleRole role, KubernetesClusterVmMapVO vmMap, + Integer sourcePortStart, Integer sourcePortEnd, Integer destinationPort) { + this.role = role; + this.vmMap = vmMap; + this.logicalRole = role.toLogicalRole(role.isVmScoped() && vmMap != null ? vmMap.getVmId() : null); + this.sourcePortStart = sourcePortStart; + this.sourcePortEnd = sourcePortEnd; + this.destinationPort = destinationPort; + } + + private void setSourcePorts(Integer sourcePortStart, Integer sourcePortEnd) { + this.sourcePortStart = sourcePortStart; + this.sourcePortEnd = sourcePortEnd; + } + } + + private static final class RequestedRule { + private final KubernetesClusterNetworkRuleAdoptionSpec spec; + private final ExpectedRule expected; + private Long resourceId; + + private RequestedRule(KubernetesClusterNetworkRuleAdoptionSpec spec, ExpectedRule expected) { + this.spec = spec; + this.expected = expected; + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java index 07d062e23a19..b930cbdacd26 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -21,13 +21,14 @@ import com.cloud.exception.ManagementServerException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; -import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.service.ServiceOfferingVO; import com.cloud.utils.Pair; @@ -37,19 +38,14 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.context.CallContext; -import javax.inject.Inject; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.Optional; public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker { - @Inject - private FirewallRulesDao firewallRulesDao; - private long removeNodeTimeoutTime; public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { @@ -64,9 +60,9 @@ public boolean removeNodesFromCluster(List nodeIds) { if (Objects.isNull(network)) { throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); } - IpAddress publicIp = null; + IpAddress publicIp; try { - publicIp = getPublicIp(network); + publicIp = getPublicIpForNodeAccess(network); } catch (ManagementServerException e) { throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); } @@ -96,7 +92,7 @@ private boolean removeNodesFromCluster(List nodeIds, Network network, IpAd continue; } try { - removeNodeVmFromCluster(nodeId, vm.getDisplayName().toLowerCase(Locale.ROOT), publicIp.getAddress().addr()); + removeNodeVmFromCluster(nodeId, vm, network, publicIp); result &= removeNodePortForwardingRules(nodeId, network, vm); if (System.currentTimeMillis() > removeNodeTimeoutTime) { logger.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName())); @@ -123,52 +119,86 @@ private boolean removeNodesFromCluster(List nodeIds, Network network, IpAd } protected boolean removeNodePortForwardingRules(Long nodeId, Network network, UserVmVO vm) { - List pfRules = portForwardingRulesDao.listByVm(nodeId); - boolean result = true; - for (PortForwardingRuleVO pfRule : pfRules) { - try { - result &= rulesService.revokePortForwardingRule(pfRule.getId(), true); - if (Objects.isNull(network.getVpcId())) { - FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); - result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + if (manager.isDirectAccess(network)) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null || !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals( + cluster.getNetworkRuleOwnershipState())) { + return true; + } + } + KubernetesClusterFirewallRuleMapVO portForwardingOwnership = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), SSH_PORT_FORWARD_ROLE_PREFIX + nodeId); + try { + if (portForwardingOwnership != null) { + deleteManagedPortForwardingRule(portForwardingOwnership); + } + if (Objects.isNull(network.getVpcId())) { + KubernetesClusterFirewallRuleMapVO externalFirewallOwnership = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + nodeId); + if (externalFirewallOwnership != null) { + deleteManagedFirewallRule(externalFirewallOwnership); } - } catch (Exception e) { - String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); - logger.error(err, e); } + return true; + } catch (Exception e) { + String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); + logger.error(err, e); + return false; } - return result; } - private void removeNodeVmFromCluster(Long nodeId, String nodeName, String publicIp) throws Exception { + private void removeNodeVmFromCluster(Long nodeId, UserVmVO vm, Network network, IpAddress publicIp) throws Exception { File removeNodeScriptFile = retrieveScriptFile(removeNodeFromClusterScript); - copyScriptFile(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, removeNodeScriptFile, removeNodeFromClusterScript); + Pair controlNodeAccess = getKubernetesClusterServerIpSshPort(null); + if (controlNodeAccess.first() == null) { + throw new ManagementServerException(String.format( + "No management endpoint was found for Kubernetes cluster %s", kubernetesCluster.getName())); + } + copyScriptFile(controlNodeAccess.first(), controlNodeAccess.second(), removeNodeScriptFile, removeNodeFromClusterScript); File pkFile = getManagementServerSshPublicKeyFile(); + String nodeName = vm.getDisplayName().toLowerCase(Locale.ROOT); String command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "remove"); - Pair result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), + Pair result = SshHelper.sshExecute(controlNodeAccess.first(), controlNodeAccess.second(), getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 10 * 60 * 1000); if (Boolean.FALSE.equals(result.first())) { logger.error(String.format("Node: %s failed to be gracefully drained as a worker node from cluster %s ", nodeName, kubernetesCluster.getName())); } - List nodePfRules = portForwardingRulesDao.listByVm(nodeId); - Optional nodeSshPort = nodePfRules.stream().filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT - && rule.getVirtualMachineId() == nodeId && rule.getSourcePortStart() >= CLUSTER_NODES_DEFAULT_START_SSH_PORT).findFirst(); - if (nodeSshPort.isPresent()) { - copyScriptFile(publicIp, nodeSshPort.get().getSourcePortStart(), removeNodeScriptFile, removeNodeFromClusterScript); - command = String.format("sudo %s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "worker", "remove"); - result = SshHelper.sshExecute(publicIp, nodeSshPort.get().getSourcePortStart(), getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 10 * 60 * 1000); - if (Boolean.FALSE.equals(result.first())) { - logger.error(String.format("Failed to reset node: %s from cluster %s ", nodeName, kubernetesCluster.getName())); - } - command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "delete"); - result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 10 * 60 * 1000); - if (Boolean.FALSE.equals(result.first())) { - logger.error(String.format("Node: %s failed to be gracefully delete node from cluster %s ", nodeName, kubernetesCluster.getName())); - } + Pair nodeAccess = getRemovedNodeIpSshPort(nodeId, network, publicIp); + copyScriptFile(nodeAccess.first(), nodeAccess.second(), removeNodeScriptFile, removeNodeFromClusterScript); + command = String.format("sudo %s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "worker", "remove"); + result = SshHelper.sshExecute(nodeAccess.first(), nodeAccess.second(), getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Failed to reset node: %s from cluster %s ", nodeName, kubernetesCluster.getName())); + } + command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "delete"); + result = SshHelper.sshExecute(controlNodeAccess.first(), controlNodeAccess.second(), getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Node: %s failed to be gracefully delete node from cluster %s ", nodeName, kubernetesCluster.getName())); + } + } + protected Pair getRemovedNodeIpSshPort(long nodeId, Network network, IpAddress publicIp) + throws ManagementServerException { + if (manager.isDirectAccess(network)) { + return getNodeIpSshPort(network, nodeId, null, DEFAULT_SSH_PORT); + } + KubernetesClusterFirewallRuleMapVO mapping = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), SSH_PORT_FORWARD_ROLE_PREFIX + nodeId); + PortForwardingRuleVO rule = mapping == null ? null : portForwardingRulesDao.findById(mapping.getFirewallRuleId()); + if (rule == null || FirewallRule.State.Revoke.equals(rule.getState()) + || !Objects.equals(rule.getSourceIpAddressId(), publicIp.getId()) + || !Objects.equals(rule.getNetworkId(), network.getId()) + || !Objects.equals(rule.getDestinationPortStart(), DEFAULT_SSH_PORT) + || !Objects.equals(rule.getVirtualMachineId(), nodeId) + || rule.getSourcePortStart() == null + || rule.getSourcePortStart() < CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + throw new ManagementServerException(String.format( + "No managed SSH port forwarding rule was found for node %d in Kubernetes cluster %s", + nodeId, kubernetesCluster.getName())); } + return getNodeIpSshPort(network, nodeId, publicIp, rule.getSourcePortStart()); } private void updateKubernetesCluster(long clusterId, List nodesRemoved, long deallocatedRam, long deallocatedCores) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index a1b2294bcd4d..40be210f9225 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -21,15 +21,18 @@ import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; -import static com.cloud.utils.db.Transaction.execute; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -41,15 +44,11 @@ import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; -import com.cloud.network.rules.RulesService; -import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.rules.FirewallManager; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.utils.db.Transaction; -import com.cloud.utils.net.Ip; import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.commons.codec.binary.Base64; @@ -78,23 +77,30 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleLifecycleState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancer; -import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.NetworkACLItemDao; import com.cloud.network.vpc.NetworkACLItemVO; import com.cloud.network.vpc.NetworkACLService; +import com.cloud.network.vpc.NetworkACLVO; +import com.cloud.network.vpc.dao.NetworkACLDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.storage.Volume; @@ -130,26 +136,24 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected HostDao hostDao; @Inject - protected FirewallRulesDao firewallRulesDao; - @Inject protected NetworkACLService networkACLService; @Inject protected NetworkACLItemDao networkACLItemDao; @Inject - protected LoadBalancingRulesService lbService; + protected NetworkACLDao networkACLDao; @Inject - protected RulesService rulesService; + protected LoadBalancingRulesService lbService; @Inject protected FirewallManager firewallManager; @Inject - protected PortForwardingRulesDao portForwardingRulesDao; - @Inject protected ResourceManager resourceManager; @Inject protected DedicatedResourceDao dedicatedResourceDao; @Inject protected LoadBalancerDao loadBalancerDao; @Inject + protected LoadBalancerVMMapDao loadBalancerVMMapDao; + @Inject protected UserVmManager userVmManager; @Inject protected LaunchPermissionDao launchPermissionDao; @@ -483,54 +487,6 @@ protected UserVm createKubernetesNode(String joinIp, Long domainId, Long account return nodeVm; } - protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, - IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { - List sourceCidrList = new ArrayList(); - sourceCidrList.add("0.0.0.0/0"); - - CreateFirewallRuleCmd firewallRule = new CreateFirewallRuleCmd(); - firewallRule = ComponentContext.inject(firewallRule); - - firewallRule.setIpAddressId(publicIp.getId()); - - firewallRule.setProtocol("TCP"); - - firewallRule.setPublicStartPort(startPort); - - firewallRule.setPublicEndPort(endPort); - - firewallRule.setSourceCidrList(sourceCidrList); - - firewallService.createIngressFirewallRule(firewallRule); - firewallService.applyIngressFwRules(publicIp.getId(), account); - } - - protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network network, Account account, - final long vmId, final int sourcePort, final int destPort) throws NetworkRuleConflictException, ResourceUnavailableException { - final long publicIpId = publicIp.getId(); - final long networkId = network.getId(); - final long accountId = account.getId(); - final long domainId = account.getDomainId(); - Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); - final Ip vmIp = new Ip(vmNic.getIPv4Address()); - PortForwardingRuleVO pfRule = execute((TransactionCallbackWithException) status -> { - PortForwardingRuleVO newRule = - new PortForwardingRuleVO(null, publicIpId, - sourcePort, sourcePort, - vmIp, - destPort, destPort, - "tcp", networkId, accountId, domainId, vmId, null); - newRule.setDisplay(true); - newRule.setState(FirewallRule.State.Add); - newRule = portForwardingRulesDao.persist(newRule); - return newRule; - }); - rulesService.applyPortForwardingRules(publicIp.getId(), account); - if (logger.isInfoEnabled()) { - logger.info("Provisioned SSH port forwarding rule: {} from port {} to {} on {} to the VM IP: {} in Kubernetes cluster: {}", pfRule, sourcePort, destPort, publicIp.getAddress().addr(), vmIp, kubernetesCluster); - } - } - /** * To provision SSH port forwarding rules for the given Kubernetes cluster * for its given virtual machines @@ -547,100 +503,110 @@ protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network n protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account, List clusterVMIds, Map vmIdPortMap) throws ResourceUnavailableException, NetworkRuleConflictException { - if (!CollectionUtils.isEmpty(clusterVMIds)) { - int defaultNodesCount = clusterVMIds.size() - vmIdPortMap.size(); - int sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT; - for (int i = 0; i < defaultNodesCount; ++i) { - sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i; - provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); + if (CollectionUtils.isEmpty(clusterVMIds)) { + return; + } + Map explicitPorts = vmIdPortMap == null ? Collections.emptyMap() : vmIdPortMap; + Set reservedPorts = new HashSet<>(); + for (Map.Entry entry : explicitPorts.entrySet()) { + Integer port = entry.getValue(); + if (port == null || port < CLUSTER_NODES_DEFAULT_START_SSH_PORT || port > NetUtils.PORT_RANGE_MAX) { + throw new NetworkRuleConflictException(String.format("Invalid SSH public port configured for VM %d", entry.getKey())); } - for (int i = defaultNodesCount; i < clusterVMIds.size(); ++i) { - sourcePort += 1; - provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); + if (!reservedPorts.add(port)) { + throw new NetworkRuleConflictException(String.format("SSH public port %d is assigned to more than one VM", port)); } } - } - protected FirewallRule removeApiFirewallRule(final IpAddress publicIp) { - FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO); - for (FirewallRuleVO firewallRule : firewallRules) { - Integer startPort = firewallRule.getSourcePortStart(); - Integer endPort = firewallRule.getSourcePortEnd(); - if (startPort != null && startPort == CLUSTER_API_PORT && - endPort != null && endPort == CLUSTER_API_PORT) { - rule = firewallRule; - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - logger.debug("The API firewall rule [%s] with the id [%s] was revoked",firewallRule.getName(),firewallRule.getId()); - break; + int nextDefaultPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT; + Set processedVmIds = new HashSet<>(); + for (Long vmId : clusterVMIds) { + if (vmId == null || !processedVmIds.add(vmId)) { + throw new NetworkRuleConflictException("Kubernetes cluster VM list contains a missing or duplicate VM id"); } + Integer sourcePort = explicitPorts.get(vmId); + if (sourcePort == null) { + while (reservedPorts.contains(nextDefaultPort)) { + nextDefaultPort++; + } + if (nextDefaultPort > NetUtils.PORT_RANGE_MAX) { + throw new NetworkRuleConflictException("No public SSH port is available for the Kubernetes cluster"); + } + sourcePort = nextDefaultPort++; + } + provisionPublicIpPortForwardingRule(publicIp, network, account, vmId, sourcePort, DEFAULT_SSH_PORT); } - return rule; } - protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) { - FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO); - for (FirewallRuleVO firewallRule : firewallRules) { - PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd()); - if (Objects.equals(firewallRule.getSourcePortStart(), CLUSTER_NODES_DEFAULT_START_SSH_PORT) || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) { - rule = firewallRule; - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - logger.debug("The SSH firewall rule {} with the id {} was revoked", firewallRule.getName(), firewallRule.getId()); - break; - } + protected void deleteManagedLoadBalancerRule(KubernetesClusterFirewallRuleMapVO mapping, Network network) + throws ResourceUnavailableException { + markManagedFirewallRulePendingDelete(mapping.getFirewallRuleId()); + LoadBalancerVO rule = loadBalancerDao.findById(mapping.getFirewallRuleId()); + if (rule != null && !lbService.deleteLoadBalancerRule(rule.getId(), true)) { + throw new ResourceUnavailableException("Failed to remove the Kubernetes cluster load balancing rule", Network.class, network.getId()); + } + FirewallRuleVO removedRule = firewallRulesDao.findByIdIncludingRemoved(mapping.getFirewallRuleId()); + if (removedRule != null && removedRule.getRemoved() == null) { + throw new ResourceUnavailableException("Kubernetes cluster load balancing rule removal was not confirmed", Network.class, network.getId()); } - return rule; + forgetManagedFirewallRule(mapping); } - protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { - if (!CollectionUtils.isEmpty(removedVMIds)) { - List pfRules = new ArrayList<>(); - List revokedRules = new ArrayList<>(); - for (Long vmId : removedVMIds) { - pfRules.addAll(portForwardingRulesDao.listByNetwork(network.getId())); - for (PortForwardingRuleVO pfRule : pfRules) { - if (pfRule.getVirtualMachineId() == vmId) { - portForwardingRulesDao.remove(pfRule.getId()); - logger.trace("Marking PF rule {} with Revoke state", pfRule); - pfRule.setState(FirewallRule.State.Revoke); - revokedRules.add(pfRule); - logger.debug("The Port forwarding rule {} with the id {} was removed.", pfRule.getName(), pfRule.getId()); - break; - } + protected void provisionVpcTierAllowPortACLRule(final Network network, int startPort, int endPorts, String logicalRole) + throws ResourceUnavailableException, NetworkRuleConflictException { + Network effectiveNetwork = networkDao.findById(network.getId()); + if (effectiveNetwork == null) { + throw new ResourceUnavailableException("Kubernetes cluster network no longer exists", Network.class, network.getId()); + } + Long networkAclId = effectiveNetwork.getNetworkACLId(); + if (networkAclId == null) { + throw new NetworkRuleConflictException(String.format("VPC tier %s does not have a network ACL attached", effectiveNetwork.getName())); + } + boolean sharedAcl = networkDao.listByAclId(networkAclId).stream() + .anyMatch(attachedNetwork -> attachedNetwork.getId() != effectiveNetwork.getId()); + if (sharedAcl) { + throw new NetworkRuleConflictException(String.format("Network ACL %d is shared by multiple VPC tiers and cannot contain Kubernetes cluster-owned rules", + networkAclId)); + } + List aclItems = networkACLItemDao.listByACL(networkAclId); + aclItems = aclItems.stream().filter(networkACLItem -> !NetworkACLItem.State.Revoke.equals(networkACLItem.getState())).collect(Collectors.toList()); + KubernetesClusterNetworkACLItemMapVO existingMapping = kubernetesClusterNetworkACLItemMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), logicalRole); + if (existingMapping != null) { + NetworkACLItemVO ownedRule = networkACLItemDao.findById(existingMapping.getNetworkAclItemId()); + boolean desiredRule = ownedRule != null && !NetworkACLItem.State.Revoke.equals(ownedRule.getState()) + && Objects.equals(ownedRule.getAclId(), networkAclId) + && Objects.equals(ownedRule.getSourcePortStart(), startPort) + && Objects.equals(ownedRule.getSourcePortEnd(), endPorts) + && NetUtils.TCP_PROTO.equalsIgnoreCase(ownedRule.getProtocol()) + && NetworkACLItem.TrafficType.Ingress.equals(ownedRule.getTrafficType()) + && NetworkACLItem.Action.Allow.equals(ownedRule.getAction()) + && ownedRule.getSourceCidrList() != null + && ownedRule.getSourceCidrList().size() == 2 + && new HashSet<>(ownedRule.getSourceCidrList()).equals(Set.of(NetUtils.ALL_IP4_CIDRS, NetUtils.ALL_IP6_CIDRS)); + if (desiredRule && existingMapping.getLifecycleState() != KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE) { + if (!networkACLService.applyNetworkACL(ownedRule.getAclId())) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster network ACL item", Network.class, network.getId()); } + activateManagedNetworkAclItem(ownedRule.getId()); + return; } - firewallManager.applyRules(revokedRules, false, true); + deleteManagedNetworkAclItem(existingMapping, effectiveNetwork); } - } - - protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, int startPort, int endPort) - throws ResourceUnavailableException { - List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); - for (PortForwardingRuleVO pfRule : pfRules) { - if (startPort <= pfRule.getSourcePortStart() && pfRule.getSourcePortStart() <= endPort) { - portForwardingRulesDao.remove(pfRule.getId()); - logger.debug("The Port forwarding rule [{}] with the id [{}] was mark as revoked.", pfRule.getName(), pfRule.getId()); - pfRule.setState(FirewallRule.State.Revoke); + for (NetworkACLItemVO aclItem : aclItems) { + List sourceCidrs = aclItem.getSourceCidrList(); + boolean desiredRule = NetUtils.TCP_PROTO.equalsIgnoreCase(aclItem.getProtocol()) + && Objects.equals(aclItem.getSourcePortStart(), startPort) + && Objects.equals(aclItem.getSourcePortEnd(), endPorts) + && NetworkACLItem.TrafficType.Ingress.equals(aclItem.getTrafficType()) + && NetworkACLItem.Action.Allow.equals(aclItem.getAction()) + && sourceCidrs != null && sourceCidrs.contains(NetUtils.ALL_IP4_CIDRS) + && sourceCidrs.contains(NetUtils.ALL_IP6_CIDRS); + if (desiredRule) { + throw new NetworkRuleConflictException(String.format("Matching network ACL item %d is not owned by Kubernetes cluster %s", + aclItem.getId(), kubernetesCluster.getName())); } } - firewallManager.applyRules(pfRules, false, true); - } - - protected void removeLoadBalancingRule(final IpAddress publicIp, final Network network, - final Account account) throws ResourceUnavailableException { - List loadBalancerRules = loadBalancerDao.listByIpAddress(publicIp.getId()); - loadBalancerRules.stream().filter(lbRules -> lbRules.getNetworkId() == network.getId() && lbRules.getAccountId() == account.getId() && lbRules.getSourcePortStart() == CLUSTER_API_PORT - && lbRules.getSourcePortEnd() == CLUSTER_API_PORT).forEach(lbRule -> { - lbService.deleteLoadBalancerRule(lbRule.getId(), true); - logger.debug("The load balancing rule with the Id: {} was removed",lbRule.getId()); - }); - } - - protected void provisionVpcTierAllowPortACLRule(final Network network, int startPort, int endPorts) throws NoSuchFieldException, - IllegalAccessException, ResourceUnavailableException { - List aclItems = networkACLItemDao.listByACL(network.getNetworkACLId()); - aclItems = aclItems.stream().filter(networkACLItem -> !NetworkACLItem.State.Revoke.equals(networkACLItem.getState())).collect(Collectors.toList()); CreateNetworkACLCmd networkACLRule = new CreateNetworkACLCmd(); networkACLRule = ComponentContext.inject(networkACLRule); @@ -652,50 +618,253 @@ protected void provisionVpcTierAllowPortACLRule(final Network network, int start networkACLRule.setTrafficType(NetworkACLItem.TrafficType.Ingress.toString()); - networkACLRule.setNetworkId(network.getId()); + networkACLRule.setNetworkId(effectiveNetwork.getId()); - networkACLRule.setAclId(network.getNetworkACLId()); + networkACLRule.setAclId(networkAclId); networkACLRule.setAction(NetworkACLItem.Action.Allow.toString()); - NetworkACLItem aclRule = networkACLService.createNetworkACLItem(networkACLRule); + CreateNetworkACLCmd finalNetworkACLRule = networkACLRule; + NetworkACLItem aclRule = Transaction.execute((TransactionCallback) status -> { + lockClusterForNetworkRuleMutation(); + NetworkACLItem rule = networkACLService.createNetworkACLItem(finalNetworkACLRule); + recordManagedNetworkAclItem(rule.getId(), logicalRole); + return rule; + }); networkACLService.moveRuleToTheTopInACLList(aclRule); - networkACLService.applyNetworkACL(aclRule.getAclId()); + if (!networkACLService.applyNetworkACL(aclRule.getAclId())) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes cluster network ACL item", Network.class, effectiveNetwork.getId()); + } + activateManagedNetworkAclItem(aclRule.getId()); } - protected void removeVpcTierAllowPortACLRule(final Network network, int startPort, int endPort) throws NoSuchFieldException, - IllegalAccessException, ResourceUnavailableException { - List aclItems = networkACLItemDao.listByACL(network.getNetworkACLId()); - aclItems = aclItems.stream().filter(networkACLItem -> (networkACLItem.getProtocol() != null && - networkACLItem.getProtocol().equals("TCP") && - networkACLItem.getSourcePortStart() != null && - networkACLItem.getSourcePortStart().equals(startPort) && - networkACLItem.getSourcePortEnd() != null && - networkACLItem.getSourcePortEnd().equals(endPort) && - networkACLItem.getAction().equals(NetworkACLItem.Action.Allow))) - .collect(Collectors.toList()); + protected void deleteManagedNetworkAclItem(KubernetesClusterNetworkACLItemMapVO mapping, Network network) + throws ResourceUnavailableException { + validateManagedNetworkAclItemMapping(mapping, network); + markManagedNetworkAclItemPendingDelete(mapping.getNetworkAclItemId()); + NetworkACLItemVO item = networkACLItemDao.findById(mapping.getNetworkAclItemId()); + if (item != null && !networkACLService.revokeNetworkACLItem(item.getId())) { + throw new ResourceUnavailableException("Failed to remove the Kubernetes cluster network ACL item", Network.class, network.getId()); + } + if (networkACLItemDao.findById(mapping.getNetworkAclItemId()) != null) { + throw new ResourceUnavailableException("Kubernetes cluster network ACL item removal was not confirmed", Network.class, network.getId()); + } + forgetManagedNetworkAclItem(mapping); + } - for (NetworkACLItemVO aclItem : aclItems) { - networkACLService.revokeNetworkACLItem(aclItem.getId()); + protected void validateManagedFirewallRuleMapping(KubernetesClusterFirewallRuleMapVO mapping, Network network) { + KubernetesClusterNetworkRuleRole role = KubernetesClusterNetworkRuleRole.fromLogicalRole(mapping.getLogicalRole()); + if (KubernetesClusterNetworkRuleRole.ResourceType.NETWORK_ACL_ITEM.equals(role.getResourceType())) { + throw new CloudRuntimeException(String.format("Network ACL role %s is stored as firewall-rule ownership", mapping.getLogicalRole())); + } + role.getVmId(mapping.getLogicalRole()); + FirewallRuleVO rule = firewallRulesDao.findById(mapping.getFirewallRuleId()); + if (rule == null) { + return; + } + FirewallRule.Purpose expectedPurpose; + switch (role.getResourceType()) { + case FIREWALL: + expectedPurpose = FirewallRule.Purpose.Firewall; + break; + case PORT_FORWARDING: + expectedPurpose = FirewallRule.Purpose.PortForwarding; + break; + case LOAD_BALANCER: + expectedPurpose = FirewallRule.Purpose.LoadBalancing; + break; + default: + throw new CloudRuntimeException(String.format("Unsupported firewall-rule resource type %s", role.getResourceType())); + } + if (!expectedPurpose.equals(rule.getPurpose()) + || !Objects.equals(rule.getNetworkId(), network.getId()) + || rule.getAccountId() != kubernetesCluster.getAccountId() + || rule.getDomainId() != kubernetesCluster.getDomainId()) { + throw new CloudRuntimeException(String.format( + "Managed rule %d for role %s does not belong to Kubernetes cluster %s; refusing deletion", + rule.getId(), mapping.getLogicalRole(), kubernetesCluster.getName())); } } + protected void validateManagedNetworkAclItemMapping(KubernetesClusterNetworkACLItemMapVO mapping, Network network) { + KubernetesClusterNetworkRuleRole role = KubernetesClusterNetworkRuleRole.fromLogicalRole(mapping.getLogicalRole()); + if (!KubernetesClusterNetworkRuleRole.ResourceType.NETWORK_ACL_ITEM.equals(role.getResourceType())) { + throw new CloudRuntimeException(String.format("Firewall-rule role %s is stored as network ACL ownership", mapping.getLogicalRole())); + } + role.getVmId(mapping.getLogicalRole()); + NetworkACLItemVO item = networkACLItemDao.findById(mapping.getNetworkAclItemId()); + if (item == null) { + return; + } + NetworkACLVO acl = networkACLDao.findById(item.getAclId()); + if (acl == null || !Objects.equals(acl.getVpcId(), network.getVpcId())) { + throw new CloudRuntimeException(String.format( + "Managed network ACL item %d for role %s does not belong to Kubernetes cluster %s VPC; refusing deletion", + item.getId(), mapping.getLogicalRole(), kubernetesCluster.getName())); + } + if (networkDao.listByAclId(item.getAclId()).stream() + .anyMatch(attachedNetwork -> attachedNetwork.getId() != network.getId())) { + throw new CloudRuntimeException(String.format( + "Managed network ACL item %d for role %s is attached to another VPC tier; refusing deletion", + item.getId(), mapping.getLogicalRole())); + } + } + + protected void deleteManagedFirewallRuleByRole(KubernetesClusterFirewallRuleMapVO mapping, Network network) + throws ResourceUnavailableException { + KubernetesClusterNetworkRuleRole role = KubernetesClusterNetworkRuleRole.fromLogicalRole(mapping.getLogicalRole()); + switch (role.getResourceType()) { + case FIREWALL: + deleteManagedFirewallRule(mapping); + break; + case PORT_FORWARDING: + deleteManagedPortForwardingRule(mapping); + break; + case LOAD_BALANCER: + deleteManagedLoadBalancerRule(mapping, network); + break; + default: + throw new CloudRuntimeException(String.format("Unsupported firewall-rule ownership role %s", mapping.getLogicalRole())); + } + } + + protected void deleteManagedNetworkRulesNotIn(Set desiredRoles, Network network) throws ManagementServerException { + List allFirewallMappings = kubernetesClusterFirewallRuleMapDao.listByClusterId(kubernetesCluster.getId()); + List allAclMappings = kubernetesClusterNetworkACLItemMapDao.listByClusterId(kubernetesCluster.getId()); + allFirewallMappings.forEach(mapping -> validateManagedFirewallRuleMapping(mapping, network)); + allAclMappings.forEach(mapping -> validateManagedNetworkAclItemMapping(mapping, network)); + List firewallMappings = allFirewallMappings.stream() + .filter(mapping -> !desiredRoles.contains(mapping.getLogicalRole())) + .sorted(Comparator.comparingLong(KubernetesClusterFirewallRuleMapVO::getId)) + .collect(Collectors.toList()); + List aclMappings = allAclMappings.stream() + .filter(mapping -> !desiredRoles.contains(mapping.getLogicalRole())) + .sorted(Comparator.comparingLong(KubernetesClusterNetworkACLItemMapVO::getId)) + .collect(Collectors.toList()); + try { + for (KubernetesClusterFirewallRuleMapVO mapping : firewallMappings) { + deleteManagedFirewallRuleByRole(mapping, network); + } + for (KubernetesClusterNetworkACLItemMapVO mapping : aclMappings) { + deleteManagedNetworkAclItem(mapping, network); + } + } catch (ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to remove stale managed network rules for Kubernetes cluster %s", + kubernetesCluster.getName()), e); + } + } + + protected void deleteAllManagedNetworkRules(Network network) throws ManagementServerException { + deleteManagedNetworkRulesNotIn(Collections.emptySet(), network); + } + protected void provisionLoadBalancerRule(final IpAddress publicIp, final Network network, final Account account, final List clusterVMIds, final int port) throws NetworkRuleConflictException, - InsufficientAddressCapacityException { - LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", - port, port, port, port, - publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(), - account.getId(), false, NetUtils.TCP_PROTO, true); - - Map> vmIdIpMap = new HashMap<>(); - for (int i = 0; i < kubernetesCluster.getControlNodeCount(); ++i) { - List ips = new ArrayList<>(); - Nic controlVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); - ips.add(controlVmNic.getIPv4Address()); - vmIdIpMap.put(clusterVMIds.get(i), ips); - } - lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, null, false); + InsufficientAddressCapacityException, ResourceUnavailableException { + LoadBalancer lb = null; + KubernetesClusterFirewallRuleMapVO existingMapping = kubernetesClusterFirewallRuleMapDao.findByClusterIdAndLogicalRole( + kubernetesCluster.getId(), API_LOAD_BALANCER_ROLE); + if (existingMapping != null) { + LoadBalancerVO ownedRule = loadBalancerDao.findById(existingMapping.getFirewallRuleId()); + boolean desiredRule = ownedRule != null && !FirewallRule.State.Revoke.equals(ownedRule.getState()) + && Objects.equals(ownedRule.getSourceIpAddressId(), publicIp.getId()) + && Objects.equals(ownedRule.getSourcePortStart(), port) + && Objects.equals(ownedRule.getSourcePortEnd(), port) + && Objects.equals(ownedRule.getDefaultPortStart(), port) + && Objects.equals(ownedRule.getDefaultPortEnd(), port) + && Objects.equals(ownedRule.getNetworkId(), network.getId()) + && ownedRule.getAccountId() == account.getId() + && ownedRule.getDomainId() == account.getDomainId() + && NetUtils.TCP_PROTO.equalsIgnoreCase(ownedRule.getProtocol()) + && NetUtils.TCP_PROTO.equalsIgnoreCase(ownedRule.getLbProtocol()) + && "api-lb".equals(ownedRule.getName()) + && "roundrobin".equalsIgnoreCase(ownedRule.getAlgorithm()); + if (desiredRule && existingMapping.getLifecycleState() != KubernetesClusterNetworkRuleLifecycleState.PENDING_DELETE) { + lb = ownedRule; + } else { + deleteManagedLoadBalancerRule(existingMapping, network); + } + } + if (lb == null) { + for (LoadBalancerVO existingRule : loadBalancerDao.listByIpAddress(publicIp.getId())) { + if (!FirewallRule.State.Revoke.equals(existingRule.getState()) + && existingRule.getSourcePortStart() <= port && existingRule.getSourcePortEnd() >= port) { + boolean desiredRule = existingRule.getSourcePortStart() == port && existingRule.getSourcePortEnd() == port + && existingRule.getDefaultPortStart() == port && existingRule.getDefaultPortEnd() == port + && Objects.equals(existingRule.getNetworkId(), network.getId()) && existingRule.getAccountId() == account.getId() + && existingRule.getDomainId() == account.getDomainId() + && NetUtils.TCP_PROTO.equalsIgnoreCase(existingRule.getProtocol()) + && NetUtils.TCP_PROTO.equalsIgnoreCase(existingRule.getLbProtocol()) + && "api-lb".equals(existingRule.getName()) && "roundrobin".equalsIgnoreCase(existingRule.getAlgorithm()); + if (desiredRule) { + throw new NetworkRuleConflictException(String.format("Matching load balancing rule %d is not owned by Kubernetes cluster %s", + existingRule.getId(), kubernetesCluster.getName())); + } + throw new NetworkRuleConflictException(String.format("A matching load balancing rule %d is not owned by Kubernetes cluster %s", + existingRule.getId(), kubernetesCluster.getName())); + } + } + } + if (lb == null) { + try { + lb = Transaction.execute((TransactionCallbackWithException) status -> { + lockClusterForNetworkRuleMutation(); + LoadBalancer created = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", + port, port, port, port, publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(), + account.getId(), false, NetUtils.TCP_PROTO, true); + recordManagedFirewallRule(created.getId(), API_LOAD_BALANCER_ROLE); + return created; + }); + } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { + throw e; + } catch (Exception e) { + throw new CloudRuntimeException("Failed to create the Kubernetes cluster load balancing rule ownership atomically", e); + } + } + + Map> desiredVmIdIpMap = new HashMap<>(); + List existingMappings = loadBalancerVMMapDao.listByLoadBalancerId(lb.getId(), false); + Map clusterVmMappings = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()).stream() + .collect(Collectors.toMap(KubernetesClusterVmMapVO::getVmId, vmMap -> vmMap)); + for (Long vmId : clusterVMIds) { + KubernetesClusterVmMapVO vmMap = clusterVmMappings.get(vmId); + if (vmMap == null || !vmMap.isControlNode()) { + continue; + } + Nic controlVmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); + if (controlVmNic == null || StringUtils.isBlank(controlVmNic.getIPv4Address())) { + throw new CloudRuntimeException(String.format( + "No IPv4 address was found for control node %d on Kubernetes cluster network %d", + vmId, kubernetesCluster.getNetworkId())); + } + desiredVmIdIpMap.put(vmId, List.of(controlVmNic.getIPv4Address())); + } + Map> staleVmIdIpMap = new HashMap<>(); + for (LoadBalancerVMMapVO existing : existingMappings) { + List desiredIps = desiredVmIdIpMap.get(existing.getInstanceId()); + if (desiredIps == null || !desiredIps.contains(existing.getInstanceIp())) { + staleVmIdIpMap.computeIfAbsent(existing.getInstanceId(), ignored -> new ArrayList<>()).add(existing.getInstanceIp()); + } + } + if (!staleVmIdIpMap.isEmpty() && !lbService.removeFromLoadBalancer(lb.getId(), null, staleVmIdIpMap, false)) { + throw new ResourceUnavailableException("Failed to remove stale control nodes from the Kubernetes API load balancing rule", Network.class, + network.getId()); + } + Map> missingVmIdIpMap = new HashMap<>(); + for (Map.Entry> desired : desiredVmIdIpMap.entrySet()) { + boolean alreadyMapped = existingMappings.stream().anyMatch(mapping -> Objects.equals(mapping.getInstanceId(), desired.getKey()) + && desired.getValue().contains(mapping.getInstanceIp()) && !staleVmIdIpMap.containsKey(mapping.getInstanceId())); + if (!alreadyMapped) { + missingVmIdIpMap.put(desired.getKey(), desired.getValue()); + } + } + if (!missingVmIdIpMap.isEmpty() && !lbService.assignToLoadBalancer(lb.getId(), null, missingVmIdIpMap, null, false)) { + throw new ResourceUnavailableException("Failed to assign control nodes to the Kubernetes API load balancing rule", Network.class, network.getId()); + } + if (missingVmIdIpMap.isEmpty() && staleVmIdIpMap.isEmpty() && !lbService.applyLoadBalancerConfig(lb.getId())) { + throw new ResourceUnavailableException("Failed to apply the Kubernetes API load balancing rule", Network.class, network.getId()); + } + activateManagedFirewallRule(lb.getId()); } protected Map createFirewallRules(IpAddress publicIp, List clusterVMIds, boolean apiRule) throws ManagementServerException { @@ -707,11 +876,11 @@ protected Map createFirewallRules(IpAddress publicIp, List // Firewall rule for API access for control node VMs CallContext.register(CallContext.current(), null); try { - provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT); + provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT, API_FIREWALL_ROLE); if (logger.isInfoEnabled()) { logger.info("Provisioned firewall rule to open up port {} on {} for Kubernetes cluster {}", CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster); } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } finally { CallContext.unregister(); @@ -746,64 +915,44 @@ protected void setupKubernetesClusterIsolatedNetworkRules(IpAddress publicIp, Ne // Load balancer rule for API access for control node VMs try { provisionLoadBalancerRule(publicIp, network, owner, clusterVMIds, CLUSTER_API_PORT); - } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { + } catch (NetworkRuleConflictException | InsufficientAddressCapacityException | ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } } protected void createVpcTierAclRules(Network network) throws ManagementServerException { - if (network.getNetworkACLId() == NetworkACL.DEFAULT_ALLOW) { + Long networkAclId = network.getNetworkACLId(); + if (networkAclId == null) { + throw new ManagementServerException(String.format("Failed to provision ACL rules for the Kubernetes cluster : %s as VPC tier %s does not have a network ACL attached", kubernetesCluster.getName(), network.getName())); + } + if (Objects.equals(networkAclId, NetworkACL.DEFAULT_ALLOW)) { return; } // ACL rule for API access for control node VMs CallContext.register(CallContext.current(), null); try { - provisionVpcTierAllowPortACLRule(network, CLUSTER_API_PORT, CLUSTER_API_PORT); + provisionVpcTierAllowPortACLRule(network, CLUSTER_API_PORT, CLUSTER_API_PORT, API_ACL_ROLE); if (logger.isInfoEnabled()) { logger.info("Provisioned ACL rule to open up port {} on {} for Kubernetes cluster {}", CLUSTER_API_PORT, publicIpAddress, kubernetesCluster); } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) { + } catch (ResourceUnavailableException | NetworkRuleConflictException | InvalidParameterValueException | PermissionDeniedException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } finally { CallContext.unregister(); } CallContext.register(CallContext.current(), null); try { - provisionVpcTierAllowPortACLRule(network, DEFAULT_SSH_PORT, DEFAULT_SSH_PORT); + provisionVpcTierAllowPortACLRule(network, DEFAULT_SSH_PORT, DEFAULT_SSH_PORT, SSH_ACL_ROLE); if (logger.isInfoEnabled()) { logger.info("Provisioned ACL rule to open up port {} on {} for Kubernetes cluster {}", DEFAULT_SSH_PORT, publicIpAddress, kubernetesCluster); } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) { + } catch (ResourceUnavailableException | NetworkRuleConflictException | InvalidParameterValueException | PermissionDeniedException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } finally { CallContext.unregister(); } } - protected void removeVpcTierAclRules(Network network) throws ManagementServerException { - if (network.getNetworkACLId() == NetworkACL.DEFAULT_ALLOW) { - return; - } - // ACL rule for API access for control node VMs - try { - removeVpcTierAllowPortACLRule(network, CLUSTER_API_PORT, CLUSTER_API_PORT); - if (logger.isInfoEnabled()) { - logger.info("Removed network ACL rule to open up port {} on {} for Kubernetes cluster {}", CLUSTER_API_PORT, publicIpAddress, kubernetesCluster); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to remove network ACL rule for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } - // ACL rule for SSH access for all node VMs - try { - removeVpcTierAllowPortACLRule(network, DEFAULT_SSH_PORT, DEFAULT_SSH_PORT); - if (logger.isInfoEnabled()) { - logger.info("Removed network ACL rule to open up port {} on {} for Kubernetes cluster {}", DEFAULT_SSH_PORT, publicIpAddress, kubernetesCluster); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to remove network ACL rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } - } - protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network network, List clusterVMIds) throws ManagementServerException { // Create ACL rules createVpcTierAclRules(network); @@ -813,7 +962,7 @@ protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network ne // Add load balancing for API access try { provisionLoadBalancerRule(publicIp, network, owner, clusterVMIds, CLUSTER_API_PORT); - } catch (InsufficientAddressCapacityException e) { + } catch (InsufficientAddressCapacityException | ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to activate API load balancing rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } } else { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 08513dbd4487..afa0bd52697c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -46,13 +46,13 @@ import com.cloud.exception.VirtualMachineMigrationException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; -import com.cloud.network.rules.FirewallRule; import com.cloud.offering.ServiceOffering; import com.cloud.storage.LaunchPermissionVO; import com.cloud.uservm.UserVm; @@ -129,22 +129,6 @@ private void scaleKubernetesClusterIsolatedNetworkRules(final List cluster if (publicIp == null) { throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); } - - // Remove existing SSH firewall rules - FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId()); - int existingFirewallRuleSourcePortEnd; - if (firewallRule == null) { - logger.warn("SSH firewall rule not found for Kubernetes cluster: {}. It may have been manually deleted or modified.", kubernetesCluster.getName()); - existingFirewallRuleSourcePortEnd = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; - } else { - existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); - } - - try { - removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, existingFirewallRuleSourcePortEnd); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, false); } @@ -153,11 +137,6 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t if (publicIp == null) { throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); } - try { - removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } // Add port forwarding rule for SSH access on each node VM try { Map vmIdPortMap = getVmPortMap(); @@ -167,15 +146,27 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t } } + void deleteStaleManagedNetworkRulesForScale(Network clusterNetwork) throws ManagementServerException { + if (manager.isDirectAccess(clusterNetwork)) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null || !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals( + cluster.getNetworkRuleOwnershipState())) { + return; + } + } + Set desiredRoles = new KubernetesClusterNetworkRuleOwnershipValidator(this) + .getExpectedLogicalRoles(clusterNetwork, getKubernetesClusterVMMaps()); + deleteManagedNetworkRulesNotIn(desiredRoles, clusterNetwork); + } + /** - * Scale network rules for an existing Kubernetes cluster while scaling it - * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. - * Also remove port forwarding rules for all virtual machines and re-create port-forwarding rule - * to forward public IP traffic to all node VMs' private IP. - * @param clusterVMIds - * @throws ManagementServerException + * Reconcile the cluster-owned network rules with the current VM mappings after scaling. + * + * @param clusterVMIds current non-etcd cluster VM IDs + * @throws ManagementServerException when the rules cannot be reconciled */ private void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { + deleteStaleManagedNetworkRulesForScale(network); if (manager.isDirectAccess(network)) { if (logger.isDebugEnabled()) logger.debug("Network: {} for Kubernetes cluster: {} is not an isolated network " + diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 308fc07223de..b035876662f1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import com.cloud.exception.InvalidParameterValueException; @@ -60,6 +61,8 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleAdoptionSpec; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; @@ -615,24 +618,24 @@ protected void setupKubernetesClusterNetworkRules(Network network, List } return; } + setupKubernetesClusterNetworkRules(network, clusterVMs, getPublicIp(network)); + } + + protected void setupKubernetesClusterNetworkRules(Network network, List clusterVMs, IpAddress publicIp) throws ManagementServerException { List clusterVMIds = clusterVMs.stream().map(UserVm::getId).collect(Collectors.toList()); if (network.getVpcId() != null) { - IpAddress publicIp = getVpcTierKubernetesPublicIp(network); - if (publicIp == null) { - throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); - } setupKubernetesClusterVpcTierRules(publicIp, network, clusterVMIds); return; } - IpAddress publicIp = getNetworkSourceNatIp(network); - if (publicIp == null) { - throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", - network.getName(), kubernetesCluster.getName())); - } setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true); } protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network network) throws ManagementServerException, ResourceUnavailableException { + IpAddress publicIp = ipAddressDao.findByIpAndDcId(kubernetesCluster.getZoneId(), publicIpAddress); + setupKubernetesEtcdNetworkRules(etcdVms, network, publicIp); + } + + protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network network, IpAddress publicIp) throws ManagementServerException, ResourceUnavailableException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { if (logger.isDebugEnabled()) { logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); @@ -640,24 +643,27 @@ protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network net } List etcdVmIds = etcdVms.stream().map(UserVm::getId).collect(Collectors.toList()); Integer startPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); - IpAddress publicIp = ipAddressDao.findByIpAndDcId(kubernetesCluster.getZoneId(), publicIpAddress); for (int i = 0; i < etcdVmIds.size(); i++) { int etcdStartPort = startPort + i; try { if (Objects.isNull(network.getVpcId())) { - provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort); - } else if (network.getNetworkACLId() != NetworkACL.DEFAULT_ALLOW) { + provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort, + ETCD_SSH_FIREWALL_ROLE_PREFIX + etcdVmIds.get(i)); + } else if (network.getNetworkACLId() == null) { + throw new ManagementServerException(String.format("Failed to provision ACL rules for etcd access for the Kubernetes cluster : %s as VPC tier %s does not have a network ACL attached", kubernetesCluster.getName(), network.getName())); + } else if (!Objects.equals(network.getNetworkACLId(), NetworkACL.DEFAULT_ALLOW)) { try { - provisionVpcTierAllowPortACLRule(network, ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_CLIENT_REQUEST_PORT); + provisionVpcTierAllowPortACLRule(network, ETCD_NODE_CLIENT_REQUEST_PORT, + ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_CLIENT_ACL_ROLE); if (logger.isInfoEnabled()) { logger.info(String.format("Provisioned ACL rule to open up port %d on %s for etcd nodes for Kubernetes cluster %s", ETCD_NODE_CLIENT_REQUEST_PORT, publicIpAddress, kubernetesCluster.getName())); } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) { + } catch (ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) { throw new ManagementServerException(String.format("Failed to provision ACL rules for etcd client access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -665,7 +671,77 @@ protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network net } } - private void startKubernetesClusterVMs(Long domainId, Long accountId) { + public boolean reconcileKubernetesClusterNetworkRules() { + owner = accountDao.findById(kubernetesCluster.getAccountId()); + if (owner == null) { + throw new CloudRuntimeException(String.format("Account for Kubernetes cluster %s cannot be found", kubernetesCluster.getName())); + } + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + throw new CloudRuntimeException(String.format("Network for Kubernetes cluster %s cannot be found", kubernetesCluster.getName())); + } + if (manager.isDirectAccess(network)) { + try { + deleteManagedNetworkRulesNotIn(Collections.emptySet(), network); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(String.format( + "Failed to remove stale managed network rules for direct-access Kubernetes cluster %s", + kubernetesCluster.getName()), e); + } + logger.debug("Network {} for Kubernetes cluster {} uses direct access and has no CloudStack-managed public network rules to reconcile", network, kubernetesCluster); + return true; + } + if (network.getVpcId() != null && network.getNetworkACLId() == null) { + throw new CloudRuntimeException(String.format("VPC tier %s for Kubernetes cluster %s has no network ACL", network.getName(), kubernetesCluster.getName())); + } + List vmMaps = getKubernetesClusterVMMaps(); + if (CollectionUtils.isEmpty(vmMaps)) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s has no mapped virtual machines", kubernetesCluster.getName())); + } + List controlVms = new ArrayList<>(); + List workerVms = new ArrayList<>(); + List etcdVms = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + UserVm vm = userVmDao.findById(vmMap.getVmId()); + if (vm == null || VirtualMachine.State.Destroyed.equals(vm.getState()) || VirtualMachine.State.Expunging.equals(vm.getState())) { + throw new CloudRuntimeException(String.format("Mapped virtual machine %d for Kubernetes cluster %s is not available", vmMap.getVmId(), kubernetesCluster.getName())); + } + if (vmMap.isEtcdNode()) { + etcdVms.add(vm); + } else if (vmMap.isControlNode()) { + controlVms.add(vm); + } else { + workerVms.add(vm); + } + } + if (controlVms.isEmpty()) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s has no mapped control node", kubernetesCluster.getName())); + } + List clusterVms = new ArrayList<>(controlVms); + clusterVms.addAll(workerVms); + + try { + Set desiredRoles = new KubernetesClusterNetworkRuleOwnershipValidator(this) + .getExpectedLogicalRoles(network, vmMaps); + deleteManagedNetworkRulesNotIn(desiredRoles, network); + IpAddress publicIp = getPublicIp(network); + publicIpAddress = publicIp.getAddress().addr(); + setupKubernetesClusterNetworkRules(network, clusterVms, publicIp); + if (!etcdVms.isEmpty()) { + setupKubernetesEtcdNetworkRules(etcdVms, network, publicIp); + } + logger.info("Reconciled CloudStack-managed network rules for Kubernetes cluster {}", kubernetesCluster); + return true; + } catch (ManagementServerException | ResourceUnavailableException e) { + throw new CloudRuntimeException(String.format("Failed to reconcile network rules for Kubernetes cluster %s", kubernetesCluster.getName()), e); + } + } + + public boolean adoptKubernetesClusterNetworkRules(List specs) { + return new KubernetesClusterNetworkRuleOwnershipValidator(this).adopt(specs); + } + + protected void startKubernetesClusterVMs(Long domainId, Long accountId) { List clusterVms = getKubernetesClusterVMs(); for (final UserVm vm : clusterVms) { if (vm == null) { @@ -689,6 +765,18 @@ private void startKubernetesClusterVMs(Long domainId, Long accountId) { } } + protected boolean reconcileManagedNetworkRulesBeforeReadiness() { + KubernetesClusterVO currentCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (currentCluster == null) { + throw new CloudRuntimeException(String.format("Kubernetes cluster %s cannot be found", kubernetesCluster.getName())); + } + if (!KubernetesCluster.ClusterType.CloudManaged.equals(currentCluster.getClusterType()) + || !KubernetesClusterNetworkRuleOwnershipState.MANAGED.equals(currentCluster.getNetworkRuleOwnershipState())) { + return true; + } + return reconcileKubernetesClusterNetworkRules(); + } + private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) { if (map.isControlNode()) { return CONTROL; @@ -733,8 +821,11 @@ private boolean isKubernetesClusterDashboardServiceRunning(final boolean onCreat return false; } - private void updateKubernetesClusterEntryEndpoint() { + protected void updateKubernetesClusterEntryEndpoint() { KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (kubernetesClusterVO == null) { + throw new CloudRuntimeException(String.format("Failed to update endpoint of the Kubernetes cluster : %s as the cluster no longer exists, it may have been removed while the operation was in progress", kubernetesCluster.getName())); + } kubernetesClusterVO.setEndpoint(String.format("https://%s:%d/", publicIpAddress, CLUSTER_API_PORT)); kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); } @@ -878,6 +969,14 @@ public boolean startStoppedKubernetesCluster(Long domainId, Long accountId) thro final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); startKubernetesClusterVMs(domainId, accountId); + try { + if (!reconcileManagedNetworkRulesBeforeReadiness()) { + throw new CloudRuntimeException("Managed network-rule reconciliation did not complete"); + } + } catch (CloudRuntimeException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s, unable to reconcile managed network rules", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } try { InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { @@ -908,6 +1007,14 @@ public boolean startStoppedKubernetesCluster(Long domainId, Long accountId) thro public boolean reconcileAlertCluster() { init(); final long startTimeoutTime = System.currentTimeMillis() + 3 * 60 * 1000; + try { + if (!reconcileManagedNetworkRulesBeforeReadiness()) { + return false; + } + } catch (CloudRuntimeException e) { + logger.warn("Failed to reconcile managed network rules while recovering Kubernetes cluster {}", kubernetesCluster, e); + return false; + } List vmMapVOList = getKubernetesClusterVMMaps(); if (CollectionUtils.isEmpty(vmMapVOList) || vmMapVOList.size() != kubernetesCluster.getTotalNodeCount()) { return false; @@ -941,8 +1048,7 @@ public boolean reconcileAlertCluster() { if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { return false; } - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + // Alert recovery is reserved by the scanner before this queued worker starts. stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDao.java new file mode 100644 index 000000000000..d57d32d82640 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.utils.db.GenericDao; + +public interface KubernetesClusterFirewallRuleMapDao extends GenericDao { + List listByClusterId(long clusterId); + + KubernetesClusterFirewallRuleMapVO findByFirewallRuleId(long firewallRuleId); + + KubernetesClusterFirewallRuleMapVO findByClusterIdAndLogicalRole(long clusterId, String logicalRole); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDaoImpl.java new file mode 100644 index 000000000000..792073c2f324 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterFirewallRuleMapDaoImpl.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class KubernetesClusterFirewallRuleMapDaoImpl extends GenericDaoBase + implements KubernetesClusterFirewallRuleMapDao { + + private final SearchBuilder clusterIdSearch; + private final SearchBuilder firewallRuleIdSearch; + private final SearchBuilder clusterRoleSearch; + + public KubernetesClusterFirewallRuleMapDaoImpl() { + clusterIdSearch = createSearchBuilder(); + clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdSearch.done(); + + firewallRuleIdSearch = createSearchBuilder(); + firewallRuleIdSearch.and("firewallRuleId", firewallRuleIdSearch.entity().getFirewallRuleId(), SearchCriteria.Op.EQ); + firewallRuleIdSearch.done(); + + clusterRoleSearch = createSearchBuilder(); + clusterRoleSearch.and("clusterId", clusterRoleSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterRoleSearch.and("logicalRole", clusterRoleSearch.entity().getLogicalRole(), SearchCriteria.Op.EQ); + clusterRoleSearch.done(); + } + + @Override + public List listByClusterId(long clusterId) { + SearchCriteria criteria = clusterIdSearch.create(); + criteria.setParameters("clusterId", clusterId); + return listBy(criteria); + } + + @Override + public KubernetesClusterFirewallRuleMapVO findByFirewallRuleId(long firewallRuleId) { + SearchCriteria criteria = firewallRuleIdSearch.create(); + criteria.setParameters("firewallRuleId", firewallRuleId); + return findOneBy(criteria); + } + + @Override + public KubernetesClusterFirewallRuleMapVO findByClusterIdAndLogicalRole(long clusterId, String logicalRole) { + SearchCriteria criteria = clusterRoleSearch.create(); + criteria.setParameters("clusterId", clusterId); + criteria.setParameters("logicalRole", logicalRole); + return findOneBy(criteria); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDao.java new file mode 100644 index 000000000000..2aa9b9bf0d1b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.utils.db.GenericDao; + +public interface KubernetesClusterNetworkACLItemMapDao extends GenericDao { + List listByClusterId(long clusterId); + + KubernetesClusterNetworkACLItemMapVO findByNetworkAclItemId(long networkAclItemId); + + KubernetesClusterNetworkACLItemMapVO findByClusterIdAndLogicalRole(long clusterId, String logicalRole); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDaoImpl.java new file mode 100644 index 000000000000..3f5d977ba34c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNetworkACLItemMapDaoImpl.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class KubernetesClusterNetworkACLItemMapDaoImpl extends GenericDaoBase + implements KubernetesClusterNetworkACLItemMapDao { + + private final SearchBuilder clusterIdSearch; + private final SearchBuilder networkAclItemIdSearch; + private final SearchBuilder clusterRoleSearch; + + public KubernetesClusterNetworkACLItemMapDaoImpl() { + clusterIdSearch = createSearchBuilder(); + clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdSearch.done(); + + networkAclItemIdSearch = createSearchBuilder(); + networkAclItemIdSearch.and("networkAclItemId", networkAclItemIdSearch.entity().getNetworkAclItemId(), SearchCriteria.Op.EQ); + networkAclItemIdSearch.done(); + + clusterRoleSearch = createSearchBuilder(); + clusterRoleSearch.and("clusterId", clusterRoleSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterRoleSearch.and("logicalRole", clusterRoleSearch.entity().getLogicalRole(), SearchCriteria.Op.EQ); + clusterRoleSearch.done(); + } + + @Override + public List listByClusterId(long clusterId) { + SearchCriteria criteria = clusterIdSearch.create(); + criteria.setParameters("clusterId", clusterId); + return listBy(criteria); + } + + @Override + public KubernetesClusterNetworkACLItemMapVO findByNetworkAclItemId(long networkAclItemId) { + SearchCriteria criteria = networkAclItemIdSearch.create(); + criteria.setParameters("networkAclItemId", networkAclItemId); + return findOneBy(criteria); + } + + @Override + public KubernetesClusterNetworkACLItemMapVO findByClusterIdAndLogicalRole(long clusterId, String logicalRole) { + SearchCriteria criteria = clusterRoleSearch.create(); + criteria.setParameters("clusterId", clusterId); + criteria.setParameters("logicalRole", logicalRole); + return findOneBy(criteria); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/cluster/AdoptKubernetesClusterNetworkRulesCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/cluster/AdoptKubernetesClusterNetworkRulesCmd.java new file mode 100644 index 000000000000..f6f5773454a3 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/cluster/AdoptKubernetesClusterNetworkRulesCmd.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.kubernetes.cluster; + +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleAdoptionSpec; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "adoptKubernetesClusterNetworkRules", + description = "Validates and records exact ownership of existing CloudStack network rules for a legacy CloudManaged Kubernetes cluster", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + since = "4.23.0", + authorized = {RoleType.Admin}) +public class AdoptKubernetesClusterNetworkRulesCmd extends BaseAsyncCmd implements AdminCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "The ID of the legacy Kubernetes cluster") + private Long id; + + @Parameter(name = ApiConstants.RULES, type = CommandType.MAP, + description = "Existing network rules to adopt. Each entry requires resourceType, role, exact resourceId, and virtualMachineId for VM-scoped roles; omit roles whose resources do not exist") + private Map> rules; + + public Long getId() { + return id; + } + + public List getRuleSpecs() { + return KubernetesClusterNetworkRuleAdoptionSpec.parse(rules); + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_ADOPT; + } + + @Override + public String getEventDescription() { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Adopting exact network-rule ownership for Kubernetes cluster ID: %s", + cluster == null ? getId() : cluster.getUuid()); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + try { + if (!kubernetesClusterService.adoptKubernetesClusterNetworkRules(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to adopt Kubernetes cluster network-rule ownership"); + } + KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (InvalidParameterValueException e) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, e.getMessage()); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java index f47bc8ee3248..cfaf86158116 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java @@ -17,9 +17,12 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterService; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -46,6 +49,7 @@ public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.NODE_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, @@ -55,6 +59,7 @@ public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd { required = true) private List nodeIds; + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = KubernetesClusterResponse.class, description = "the ID of the Kubernetes cluster", since = "4.21.0") @@ -130,4 +135,15 @@ public Long getApiResourceId() { return getClusterId(); } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getClusterId()); + return cluster == null ? null : cluster.getNetworkId(); + } + } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 8d64fee26206..da82407b3088 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; @@ -432,6 +433,17 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = getEntityId() == null ? null : kubernetesClusterService.findById(getEntityId()); + return cluster == null ? getNetworkId() : cluster.getNetworkId(); + } + @Override public void execute() { try { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index 10a5bcc3cb1d..184d165f43b5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -19,6 +19,8 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -50,6 +52,7 @@ public class DeleteKubernetesClusterCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, @@ -124,6 +127,17 @@ public Long getApiResourceId() { return getId(); } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + @Override public String getEventDescription() { return "Deleting Kubernetes cluster with ID: " + getResourceUuid(ApiConstants.ID); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ReconcileKubernetesClusterNetworkRulesCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ReconcileKubernetesClusterNetworkRulesCmd.java new file mode 100644 index 000000000000..456ace465153 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ReconcileKubernetesClusterNetworkRulesCmd.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "reconcileKubernetesClusterNetworkRules", + description = "Recreates missing CloudStack-managed network rules for a running CloudManaged Kubernetes cluster without deleting user-managed rules", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ReconcileKubernetesClusterNetworkRulesCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "The ID of the Kubernetes cluster") + private Long id; + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NETWORK_RULES_RECONCILE; + } + + @Override + public String getEventDescription() { + String description = "Reconciling Kubernetes cluster network rules"; + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + if (cluster != null) { + description += String.format(" ID: %s", cluster.getUuid()); + } else { + description += String.format(" ID: %d", getId()); + } + return description; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + try { + if (!kubernetesClusterService.reconcileKubernetesClusterNetworkRules(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to reconcile Kubernetes cluster network rules"); + } + KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java index 1eae69377c47..dd8ee4b95ca2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java @@ -21,9 +21,12 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -52,6 +55,7 @@ public class RemoveNodesFromKubernetesClusterCmd extends BaseAsyncCmd { protected static final Logger LOGGER = LogManager.getLogger(RemoveNodesFromKubernetesClusterCmd.class); + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.NODE_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, @@ -61,6 +65,7 @@ public class RemoveNodesFromKubernetesClusterCmd extends BaseAsyncCmd { required = true) private List nodeIds; + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = KubernetesClusterResponse.class, description = "the ID of the Kubernetes cluster") @@ -122,4 +127,15 @@ public ApiCommandResourceType getApiResourceType() { public Long getApiResourceId() { return getClusterId(); } + + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getClusterId()); + return cluster == null ? null : cluster.getNetworkId(); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 1cff2649428d..46b64733831e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -62,6 +62,7 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = KubernetesClusterResponse.class, description = "The ID of the Kubernetes cluster") @@ -164,6 +165,17 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index dae367548190..f35ceae6355e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -22,6 +22,8 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -54,6 +56,7 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, required = true, description = "The ID of the Kubernetes cluster") @@ -94,6 +97,17 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index 5abeb27f59b1..8c7f8298c2f5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -19,6 +19,8 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -52,6 +54,7 @@ public class StopKubernetesClusterCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, required = true, description = "The ID of the Kubernetes cluster") @@ -92,6 +95,17 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index fb576e9ef825..77b17487d8f6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -20,6 +20,8 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -53,6 +55,7 @@ public class UpgradeKubernetesClusterCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, required = true, description = "The ID of the Kubernetes cluster") @@ -95,6 +98,17 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public String getSyncObjType() { + return BaseAsyncCmd.networkSyncObject; + } + + @Override + public Long getSyncObjId() { + KubernetesCluster cluster = kubernetesClusterService.findById(getId()); + return cluster == null ? null : cluster.getNetworkId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 932d722de354..f2bd85767141 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.api.EntityReference; import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -216,6 +217,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "The type of the cluster") private KubernetesCluster.ClusterType clusterType; + @SerializedName(ApiConstants.NETWORK_RULE_OWNERSHIP_STATE) + @Param(description = "The ownership state of network rules managed for the Kubernetes cluster", since = "4.23.0") + private KubernetesClusterNetworkRuleOwnershipState networkRuleOwnershipState; + @SerializedName(ApiConstants.CREATED) @Param(description = "The date when this Kubernetes cluster was created") private Date created; @@ -548,6 +553,14 @@ public void setClusterType(KubernetesCluster.ClusterType clusterType) { this.clusterType = clusterType; } + public KubernetesClusterNetworkRuleOwnershipState getNetworkRuleOwnershipState() { + return networkRuleOwnershipState; + } + + public void setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState networkRuleOwnershipState) { + this.networkRuleOwnershipState = networkRuleOwnershipState; + } + public void setCniConfigId(String cniConfigId) { this.cniConfigId = cniConfigId; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 053366786292..072b32e16578 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -32,6 +32,8 @@ + + diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 1fab5420c3c3..83946a0117a4 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -19,6 +19,7 @@ package com.cloud.kubernetes.cluster; +import com.cloud.api.ApiAsyncJobDispatcher; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.cpu.CPU; @@ -26,11 +27,14 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.network.Network; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; @@ -46,23 +50,32 @@ import com.cloud.utils.Pair; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import org.apache.cloudstack.affinity.AffinityGroupVO; import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.cluster.AdoptKubernetesClusterNetworkRulesCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ReconcileKubernetesClusterNetworkRulesCmd; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.commons.collections.MapUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -100,6 +113,9 @@ public class KubernetesClusterManagerImplTest { @Mock KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + NetworkDao networkDao; + @Mock VMInstanceDao vmInstanceDao; @@ -118,21 +134,142 @@ public class KubernetesClusterManagerImplTest { @Mock private HostDao hostDao; + @Mock + private AsyncJobManager asyncJobManager; + + @Mock + private ApiAsyncJobDispatcher apiAsyncJobDispatcher; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; + @Test(expected = InvalidParameterValueException.class) + public void topologyMutationRejectsLegacyUnmanagedIsolatedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getNetworkId()).thenReturn(2L); + Mockito.when(cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.doReturn(false).when(kubernetesClusterManager).isDirectAccess(network); + + kubernetesClusterManager.validateNetworkRuleOwnershipForTopologyMutation(cluster); + } + + @Test + public void topologyMutationAllowsManagedIsolatedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getNetworkId()).thenReturn(2L); + Mockito.when(cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.doReturn(false).when(kubernetesClusterManager).isDirectAccess(network); + + kubernetesClusterManager.validateNetworkRuleOwnershipForTopologyMutation(cluster); + } + + @Test + public void topologyMutationAllowsLegacyDirectAccessCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getNetworkId()).thenReturn(2L); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.doReturn(true).when(kubernetesClusterManager).isDirectAccess(network); + + kubernetesClusterManager.validateNetworkRuleOwnershipForTopologyMutation(cluster); + + Mockito.verify(cluster, Mockito.never()).getNetworkRuleOwnershipState(); + } + + @Test + public void nodeTopologyAccessValidationChecksClusterAndEveryNode() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + VMInstanceVO firstNode = Mockito.mock(VMInstanceVO.class); + VMInstanceVO secondNode = Mockito.mock(VMInstanceVO.class); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(firstNode); + Mockito.when(vmInstanceDao.findById(11L)).thenReturn(secondNode); + + kubernetesClusterManager.validateAccessToClusterAndNodes(cluster, List.of(10L, 11L)); + + Mockito.verify(accountManager).checkAccess(Mockito.any(Account.class), + Mockito.eq(org.apache.cloudstack.acl.SecurityChecker.AccessType.OperateEntry), Mockito.eq(false), Mockito.eq(cluster)); + Mockito.verify(accountManager).checkAccess(Mockito.any(Account.class), + Mockito.eq(org.apache.cloudstack.acl.SecurityChecker.AccessType.OperateEntry), Mockito.eq(false), Mockito.eq(firstNode)); + Mockito.verify(accountManager).checkAccess(Mockito.any(Account.class), + Mockito.eq(org.apache.cloudstack.acl.SecurityChecker.AccessType.OperateEntry), Mockito.eq(false), Mockito.eq(secondNode)); + } + + @Test(expected = InvalidParameterValueException.class) + public void nodeTopologyAccessValidationRejectsMissingNode() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(null); + + kubernetesClusterManager.validateAccessToClusterAndNodes(cluster, List.of(10L)); + } + + @Test(expected = PermissionDeniedException.class) + public void addNodesRejectsCrossTenantClusterBeforeTopologyValidation() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + AddNodesToKubernetesClusterCmd cmd = Mockito.mock(AddNodesToKubernetesClusterCmd.class); + Mockito.when(cmd.getClusterId()).thenReturn(1L); + Mockito.when(cmd.getNodeIds()).thenReturn(List.of(10L)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.doThrow(new PermissionDeniedException("denied")).when(accountManager).checkAccess( + Mockito.any(Account.class), Mockito.any(), Mockito.eq(false), Mockito.eq(cluster)); + + kubernetesClusterManager.addNodesToKubernetesCluster(cmd); + } + + @Test(expected = PermissionDeniedException.class) + public void removeNodesRejectsCrossTenantNodeBeforeWorkerMutation() throws Exception { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + VMInstanceVO node = Mockito.mock(VMInstanceVO.class); + RemoveNodesFromKubernetesClusterCmd cmd = Mockito.mock(RemoveNodesFromKubernetesClusterCmd.class); + Mockito.when(cmd.getClusterId()).thenReturn(1L); + Mockito.when(cmd.getNodeIds()).thenReturn(List.of(10L)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(node); + Mockito.doThrow(new PermissionDeniedException("denied")).when(accountManager).checkAccess( + Mockito.any(Account.class), Mockito.any(), Mockito.eq(false), Mockito.eq(node)); + + kubernetesClusterManager.removeNodesFromKubernetesCluster(cmd); + } + + @Test + public void newCloudManagedClusterStartsWithManagedNetworkRuleOwnership() { + KubernetesClusterVO cluster = createClusterVo(KubernetesCluster.ClusterType.CloudManaged); + + Assert.assertEquals(KubernetesClusterNetworkRuleOwnershipState.MANAGED, + cluster.getNetworkRuleOwnershipState()); + } + @Test - public void testValidateVpcTierAllocated() { + public void newExternalManagedClusterDoesNotClaimNetworkRuleOwnership() { + KubernetesClusterVO cluster = createClusterVo(KubernetesCluster.ClusterType.ExternalManaged); + + Assert.assertEquals(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED, + cluster.getNetworkRuleOwnershipState()); + } + + private KubernetesClusterVO createClusterVo(KubernetesCluster.ClusterType clusterType) { + return new KubernetesClusterVO("cluster", "cluster", 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, KubernetesCluster.State.Created, null, 1L, 1L, + 8L, "", clusterType); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateVpcTierNoAclAttached() { Network network = Mockito.mock(Network.class); - Mockito.when(network.getState()).thenReturn(Network.State.Allocated); + Mockito.when(network.getNetworkACLId()).thenReturn(null); kubernetesClusterManager.validateVpcTier(network); } @Test(expected = InvalidParameterValueException.class) public void testValidateVpcTierDefaultDenyRule() { Network network = Mockito.mock(Network.class); - Mockito.when(network.getState()).thenReturn(Network.State.Implemented); Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_DENY); kubernetesClusterManager.validateVpcTier(network); } @@ -140,7 +277,6 @@ public void testValidateVpcTierDefaultDenyRule() { @Test public void testValidateVpcTierValid() { Network network = Mockito.mock(Network.class); - Mockito.when(network.getState()).thenReturn(Network.State.Implemented); Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_ALLOW); kubernetesClusterManager.validateVpcTier(network); } @@ -326,6 +462,213 @@ public void removeVmsFromCluster() { Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0); } + @Test(expected = InvalidParameterValueException.class) + public void reconcileNetworkRulesRejectsMissingCluster() { + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(null); + + kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void reconcileNetworkRulesRejectsExternallyManagedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + + kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void reconcileNetworkRulesRejectsClusterThatIsNotRunning() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Stopped); + + kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = PermissionDeniedException.class) + public void reconcileNetworkRulesEnforcesClusterAccessBeforeStartingWorker() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.doThrow(new PermissionDeniedException("denied")).when(accountManager).checkAccess( + Mockito.any(Account.class), Mockito.any(), Mockito.anyBoolean(), Mockito.eq(cluster)); + + kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void reconcileNetworkRulesRejectsLegacyUnmanagedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + + kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd); + } + + @Test + public void reconcileNetworkRulesRunsForAccessibleRunningCloudManagedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + KubernetesClusterStartWorker worker = Mockito.mock(KubernetesClusterStartWorker.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.doReturn(worker).when(kubernetesClusterManager).createKubernetesClusterStartWorker(cluster); + Mockito.when(worker.reconcileKubernetesClusterNetworkRules()).thenReturn(true); + + Assert.assertTrue(kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd)); + + Mockito.verify(accountManager).checkAccess(Mockito.any(Account.class), Mockito.any(), Mockito.eq(false), Mockito.eq(cluster)); + Mockito.verify(worker).reconcileKubernetesClusterNetworkRules(); + } + + @Test + public void systemAlertReconciliationRunsAlertRecoveryInsideQueuedJob() { + CallContext.unregister(); + User systemUser = Mockito.mock(User.class); + Mockito.when(systemUser.getId()).thenReturn(User.UID_SYSTEM); + CallContext.register(systemUser, Mockito.mock(Account.class)); + + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + KubernetesClusterStartWorker worker = Mockito.mock(KubernetesClusterStartWorker.class); + ReconcileKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(ReconcileKubernetesClusterNetworkRulesCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(ReconcileKubernetesClusterNetworkRulesCmd.class)); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Recovering); + Mockito.doReturn(worker).when(kubernetesClusterManager).createKubernetesClusterStartWorker(cluster); + Mockito.when(worker.reconcileAlertCluster()).thenReturn(true); + + Assert.assertTrue(kubernetesClusterManager.reconcileKubernetesClusterNetworkRules(cmd)); + + Mockito.verify(worker).reconcileAlertCluster(); + Mockito.verify(worker, Mockito.never()).reconcileKubernetesClusterNetworkRules(); + Mockito.verify(cluster, Mockito.never()).getNetworkRuleOwnershipState(); + } + + @Test + public void alertRecoveryUsesSameNetworkQueueAndRejectsDuplicateScheduling() { + long clusterId = 1L; + long networkId = 2L; + long accountId = 3L; + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(cluster.getId()).thenReturn(clusterId); + Mockito.when(cluster.getNetworkId()).thenReturn(networkId); + Mockito.when(cluster.getAccountId()).thenReturn(accountId); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Alert)) + .thenReturn(List.of(cluster)); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(clusterId)).thenReturn(List.of(vmMap)); + Mockito.when(vmMap.getVmId()).thenReturn(10L); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(10L)).thenReturn(vm); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(apiAsyncJobDispatcher.getName()).thenReturn("ApiAsyncJobDispatcher"); + Mockito.doReturn(true, false).when(kubernetesClusterManager).stateTransitTo( + clusterId, KubernetesCluster.Event.RecoveryRequested); + Mockito.doReturn(44L).when(kubernetesClusterManager).createAlertClusterReconciliationEvent(cluster); + Mockito.when(asyncJobManager.submitAsyncJob(Mockito.any(AsyncJob.class), + Mockito.eq(BaseAsyncCmd.networkSyncObject), Mockito.eq(networkId))).thenReturn(99L); + + KubernetesClusterManagerImpl.KubernetesClusterStatusScanner scanner = + kubernetesClusterManager.new KubernetesClusterStatusScanner(); + scanner.reallyRun(); + scanner.reallyRun(); + + ArgumentCaptor jobCaptor = ArgumentCaptor.forClass(AsyncJob.class); + Mockito.verify(asyncJobManager, Mockito.times(1)).submitAsyncJob(jobCaptor.capture(), + Mockito.eq(BaseAsyncCmd.networkSyncObject), Mockito.eq(networkId)); + Assert.assertEquals(ReconcileKubernetesClusterNetworkRulesCmd.class.getName(), jobCaptor.getValue().getCmd()); + Assert.assertEquals(Long.valueOf(clusterId), jobCaptor.getValue().getInstanceId()); + Assert.assertTrue(jobCaptor.getValue().getCmdInfo().contains("\"ctxStartEventId\":\"44\"")); + Mockito.verify(kubernetesClusterManager, Mockito.never()).createKubernetesClusterStartWorker(cluster); + } + + @Test(expected = PermissionDeniedException.class) + public void adoptNetworkRulesRejectsNonRootCaller() { + AdoptKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(AdoptKubernetesClusterNetworkRulesCmd.class); + Mockito.when(accountManager.isRootAdmin(Mockito.anyLong())).thenReturn(false); + + kubernetesClusterManager.adoptKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptNetworkRulesRejectsMissingCluster() { + AdoptKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(AdoptKubernetesClusterNetworkRulesCmd.class); + Mockito.when(accountManager.isRootAdmin(Mockito.anyLong())).thenReturn(true); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(null); + + kubernetesClusterManager.adoptKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptNetworkRulesRejectsExternallyManagedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + AdoptKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(AdoptKubernetesClusterNetworkRulesCmd.class); + Mockito.when(accountManager.isRootAdmin(Mockito.anyLong())).thenReturn(true); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + + kubernetesClusterManager.adoptKubernetesClusterNetworkRules(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptNetworkRulesRejectsTransientClusterState() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + AdoptKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(AdoptKubernetesClusterNetworkRulesCmd.class); + Mockito.when(accountManager.isRootAdmin(Mockito.anyLong())).thenReturn(true); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Starting); + + kubernetesClusterManager.adoptKubernetesClusterNetworkRules(cmd); + } + + @Test + public void adoptNetworkRulesRunsForRootOnStableCloudManagedCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + KubernetesClusterStartWorker worker = Mockito.mock(KubernetesClusterStartWorker.class); + AdoptKubernetesClusterNetworkRulesCmd cmd = Mockito.mock(AdoptKubernetesClusterNetworkRulesCmd.class); + List specs = Collections.emptyList(); + Mockito.when(accountManager.isRootAdmin(Mockito.anyLong())).thenReturn(true); + Mockito.when(cmd.getId()).thenReturn(1L); + Mockito.when(cmd.getRuleSpecs()).thenReturn(specs); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Stopped); + Mockito.doReturn(worker).when(kubernetesClusterManager).createKubernetesClusterStartWorker(cluster); + Mockito.when(worker.adoptKubernetesClusterNetworkRules(specs)).thenReturn(true); + + Assert.assertTrue(kubernetesClusterManager.adoptKubernetesClusterNetworkRules(cmd)); + + Mockito.verify(worker).adoptKubernetesClusterNetworkRules(specs); + } + @Test public void testValidateServiceOfferingNodeType() { Map map = new HashMap<>(); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpecTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpecTest.java new file mode 100644 index 000000000000..17a50c7dc5f9 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterNetworkRuleAdoptionSpecTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.exception.CloudRuntimeException; + +import org.junit.Assert; +import org.junit.Test; + +public class KubernetesClusterNetworkRuleAdoptionSpecTest { + + @Test + public void parseAcceptsCaseInsensitiveFieldsAndValues() { + Map declaration = new LinkedHashMap<>(); + declaration.put("ResourceType", "port_forwarding"); + declaration.put("ROLE", "ssh_port_forward"); + declaration.put("resourceId", "rule-uuid"); + declaration.put("virtualMachineId", "vm-uuid"); + + List result = KubernetesClusterNetworkRuleAdoptionSpec.parse( + Map.of("rule", declaration)); + + Assert.assertEquals(1, result.size()); + KubernetesClusterNetworkRuleAdoptionSpec spec = result.get(0); + Assert.assertEquals(KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, spec.getResourceType()); + Assert.assertEquals(KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, spec.getRole()); + Assert.assertEquals("rule-uuid", spec.getResourceUuid()); + Assert.assertEquals("vm-uuid", spec.getVirtualMachineUuid()); + } + + @Test + public void parseAcceptsEmptyManifestForDirectNetwork() { + Assert.assertTrue(KubernetesClusterNetworkRuleAdoptionSpec.parse(null).isEmpty()); + Assert.assertTrue(KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of()).isEmpty()); + } + + @Test(expected = InvalidParameterValueException.class) + public void parseRejectsFieldsRepeatedWithDifferentCase() { + Map declaration = new LinkedHashMap<>(); + declaration.put("role", "API_FIREWALL"); + declaration.put("ROLE", "API_FIREWALL"); + declaration.put("resourcetype", "FIREWALL"); + declaration.put("resourceid", "rule-uuid"); + KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of("rule", declaration)); + } + + @Test(expected = InvalidParameterValueException.class) + public void parseRejectsDuplicateLogicalIdentity() { + Map declaration = Map.of( + "role", "SSH_PORT_FORWARD", + "resourcetype", "PORT_FORWARDING", + "resourceid", "rule-uuid", + "virtualmachineid", "vm-uuid"); + KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of("first", declaration, "second", declaration)); + } + + @Test(expected = InvalidParameterValueException.class) + public void parseRejectsResourceTypeThatDoesNotMatchRole() { + KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of("rule", Map.of( + "role", "API_FIREWALL", + "resourcetype", "LOAD_BALANCER", + "resourceid", "rule-uuid"))); + } + + @Test(expected = InvalidParameterValueException.class) + public void parseRejectsDeclarationWithoutResource() { + KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of("rule", Map.of( + "role", "API_FIREWALL", + "resourcetype", "FIREWALL"))); + } + + @Test(expected = InvalidParameterValueException.class) + public void parseRejectsVmForClusterScopedRole() { + KubernetesClusterNetworkRuleAdoptionSpec.parse(Map.of("rule", Map.of( + "role", "API_FIREWALL", + "resourcetype", "FIREWALL", + "resourceid", "rule-uuid", + "virtualmachineid", "vm-uuid"))); + } + + @Test(expected = CloudRuntimeException.class) + public void logicalRoleRejectsNonPositiveVmId() { + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.toLogicalRole(0L); + } + + @Test(expected = CloudRuntimeException.class) + public void logicalRoleParserRejectsNonPositiveVmId() { + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.getVmId("SSH_PORT_FORWARD:-1"); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index a25ec55cc04a..6adc8c4a4d51 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.cloudstack.affinity.AffinityGroupVO; @@ -36,6 +37,7 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; @@ -44,8 +46,12 @@ import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; import com.cloud.network.Network; +import com.cloud.network.NetworkModel; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterActionWorkerTest { @@ -74,6 +80,9 @@ public class KubernetesClusterActionWorkerTest { @Mock AffinityGroupDao affinityGroupDao; + @Mock + NetworkModel networkModel; + KubernetesClusterActionWorker actionWorker = null; final static Long DEFAULT_ID = 1L; @@ -90,6 +99,7 @@ public void setUp() throws Exception { actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager); actionWorker.ipAddressDao = ipAddressDao; actionWorker.affinityGroupDao = affinityGroupDao; + actionWorker.networkModel = networkModel; } @Test @@ -147,6 +157,46 @@ public void testGetVpcTierKubernetesPublicIpValid() { Assert.assertNotNull(result); } + @Test + public void directNodeAccessUsesClusterNicAddressAndSshPort() throws Exception { + Network network = Mockito.mock(Network.class); + Nic nic = Mockito.mock(Nic.class); + Mockito.when(kubernetesClusterManager.isDirectAccess(network)).thenReturn(true); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(networkModel.getNicInNetwork(3L, 2L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("192.0.2.10"); + + Pair endpoint = actionWorker.getNodeIpSshPort(network, 3L, null, 2222); + + Assert.assertEquals("192.0.2.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(KubernetesClusterActionWorker.DEFAULT_SSH_PORT), endpoint.second()); + } + + @Test + public void nonDirectNodeAccessUsesPublicAddressAndForwardedPort() throws Exception { + Network network = Mockito.mock(Network.class); + IpAddress publicIp = Mockito.mock(IpAddress.class); + Mockito.when(kubernetesClusterManager.isDirectAccess(network)).thenReturn(false); + Mockito.when(publicIp.getAddress()).thenReturn(new Ip("203.0.113.10")); + + Pair endpoint = actionWorker.getNodeIpSshPort(network, 3L, publicIp, 2222); + + Assert.assertEquals("203.0.113.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(2222), endpoint.second()); + Mockito.verifyNoInteractions(networkModel); + } + + @Test + public void directNodeAccessDoesNotResolveACloudStackPublicIp() throws Exception { + KubernetesClusterActionWorker spy = Mockito.spy(actionWorker); + Network network = Mockito.mock(Network.class); + Mockito.when(kubernetesClusterManager.isDirectAccess(network)).thenReturn(true); + + Assert.assertNull(spy.getPublicIpForNodeAccess(network)); + + Mockito.verify(spy, Mockito.never()).getPublicIp(Mockito.any()); + } + @Test public void testGetAffinityGroupIdsForNodeTypeReturnsIds() { Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) @@ -229,4 +279,20 @@ public void testGetMergedAffinityGroupIdsExplicitDedicationAlreadyInList() { Assert.assertTrue(result.contains(99L)); Assert.assertTrue(result.contains(2L)); } + + @Test + public void getVmPortMapPlacesExternalNodesAfterStandardNodesAndExcludesEtcdNodes() { + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(DEFAULT_ID, 10L, true); + KubernetesClusterVmMapVO worker = new KubernetesClusterVmMapVO(DEFAULT_ID, 11L, false); + KubernetesClusterVmMapVO etcd = new KubernetesClusterVmMapVO(DEFAULT_ID, 12L, false); + etcd.setEtcdNode(true); + KubernetesClusterVmMapVO external = new KubernetesClusterVmMapVO(DEFAULT_ID, 13L, false); + external.setExternalNode(true); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(DEFAULT_ID)) + .thenReturn(List.of(control, worker, etcd, external)); + + Map vmPorts = actionWorker.getVmPortMap(); + + Assert.assertEquals(Map.of(13L, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + 2), vmPorts); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorkerTest.java new file mode 100644 index 000000000000..86f94e5bc480 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorkerTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterAddWorkerTest { + + @Mock + private KubernetesCluster cluster; + @Mock + private KubernetesClusterManagerImpl manager; + @Mock + private NetworkModel networkModel; + @Mock + private Network network; + @Mock + private IpAddress publicIp; + @Mock + private Account account; + + private KubernetesClusterAddWorker worker; + + @Before + public void setUp() { + worker = Mockito.spy(new KubernetesClusterAddWorker(cluster, manager)); + worker.networkModel = networkModel; + worker.owner = account; + } + + @Test + public void directNodeAccessUsesClusterNicWithoutCreatingPublicRules() throws Exception { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(networkModel.getNicInNetwork(3L, 2L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("192.0.2.10"); + + Pair endpoint = worker.prepareNodeAccess(network, null, account, 3L, 2222); + + Assert.assertEquals("192.0.2.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(KubernetesClusterActionWorker.DEFAULT_SSH_PORT), endpoint.second()); + Mockito.verify(worker, Mockito.never()).provisionFirewallRules(Mockito.any(), Mockito.any(), + Mockito.anyInt(), Mockito.anyInt(), Mockito.anyString()); + Mockito.verify(worker, Mockito.never()).provisionPublicIpPortForwardingRule(Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void isolatedNodeAccessPreservesFirewallAndPortForwardingPath() throws Exception { + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(null); + Mockito.when(publicIp.getAddress()).thenReturn(new Ip("203.0.113.10")); + Mockito.doNothing().when(worker).provisionFirewallRules(publicIp, account, 2222, 2222, + KubernetesClusterActionWorker.EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + 3L); + Mockito.doNothing().when(worker).provisionPublicIpPortForwardingRule(publicIp, network, account, 3L, + 2222, KubernetesClusterActionWorker.DEFAULT_SSH_PORT); + + Pair endpoint = worker.prepareNodeAccess(network, publicIp, account, 3L, 2222); + + Assert.assertEquals("203.0.113.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(2222), endpoint.second()); + Mockito.verify(worker).provisionFirewallRules(publicIp, account, 2222, 2222, + KubernetesClusterActionWorker.EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + 3L); + Mockito.verify(worker).provisionPublicIpPortForwardingRule(publicIp, network, account, 3L, 2222, + KubernetesClusterActionWorker.DEFAULT_SSH_PORT); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorkerTest.java new file mode 100644 index 000000000000..8299f0a136ce --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorkerTest.java @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.List; + +import com.cloud.exception.ManagementServerException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNetworkACLItemMapDao; +import com.cloud.network.NetworkService; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.RemoteAccessVpnVO; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.Site2SiteVpnGatewayVO; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.utils.net.Ip; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterDestroyWorkerTest { + + @Mock + private KubernetesCluster cluster; + @Mock + private KubernetesClusterManagerImpl manager; + @Mock + private KubernetesClusterFirewallRuleMapDao firewallRuleMapDao; + @Mock + private KubernetesClusterNetworkACLItemMapDao aclItemMapDao; + @Mock + private KubernetesClusterDetailsDao clusterDetailsDao; + @Mock + private NetworkDao networkDao; + @Mock + private NetworkVO network; + @Mock + private NetworkService networkService; + @Mock + private FirewallRulesDao firewallRulesDao; + @Mock + private RemoteAccessVpnDao remoteAccessVpnDao; + @Mock + private Site2SiteVpnGatewayDao site2SiteVpnGatewayDao; + private IPAddressVO publicIp; + + private KubernetesClusterDestroyWorker worker; + + @Before + public void setUp() { + manager.kubernetesClusterDetailsDao = clusterDetailsDao; + worker = Mockito.spy(new KubernetesClusterDestroyWorker(cluster, manager)); + worker.kubernetesClusterFirewallRuleMapDao = firewallRuleMapDao; + worker.kubernetesClusterNetworkACLItemMapDao = aclItemMapDao; + worker.networkService = networkService; + worker.firewallRulesDao = firewallRulesDao; + worker.remoteAccessVpnDao = remoteAccessVpnDao; + worker.site2SiteVpnGatewayDao = site2SiteVpnGatewayDao; + worker.networkDao = networkDao; + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getName()).thenReturn("cluster"); + } + + @Test + public void managedClusterDeletesExactRulesForAnyNetworkType() throws ManagementServerException { + Mockito.doNothing().when(worker).deleteAllManagedNetworkRules(network); + + worker.deleteManagedNetworkRulesIfPresent(network, KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(worker).deleteAllManagedNetworkRules(network); + } + + @Test + public void legacyClusterNeverDeletesUnownedRules() throws ManagementServerException { + worker.deleteManagedNetworkRulesIfPresent(network, KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + + Mockito.verify(worker, Mockito.never()).deleteAllManagedNetworkRules(Mockito.any()); + Mockito.verifyNoInteractions(firewallRuleMapDao, aclItemMapDao); + } + + @Test(expected = ManagementServerException.class) + public void missingNetworkWithOwnedRulesFailsClosed() throws ManagementServerException { + Mockito.when(firewallRuleMapDao.listByClusterId(1L)) + .thenReturn(List.of(Mockito.mock(KubernetesClusterFirewallRuleMapVO.class))); + Mockito.when(aclItemMapDao.listByClusterId(1L)).thenReturn(List.of()); + + worker.deleteManagedNetworkRulesIfPresent(null, KubernetesClusterNetworkRuleOwnershipState.MANAGED); + } + + @Test + public void missingNetworkWithoutOwnedRulesIsAlreadyClean() throws ManagementServerException { + Mockito.when(firewallRuleMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(aclItemMapDao.listByClusterId(1L)).thenReturn(List.of()); + + worker.deleteManagedNetworkRulesIfPresent(null, KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(worker, Mockito.never()).deleteAllManagedNetworkRules(Mockito.any()); + } + + private void prepareVpcPublicIp() { + publicIp = Mockito.spy(new IPAddressVO(new Ip("203.0.113.10"), 1L, 1L, 1L, false)); + Mockito.when(cluster.getNetworkId()).thenReturn(2L); + Mockito.when(network.getVpcId()).thenReturn(3L); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.doReturn(publicIp).when(worker).getVpcTierKubernetesPublicIp(network); + Mockito.doReturn(4L).when(publicIp).getId(); + } + + @Test + public void managedVpcPublicIpIsReleasedOnlyAfterAllResourcesAreGone() throws Exception { + prepareVpcPublicIp(); + Mockito.when(firewallRulesDao.listByIpAndNotRevoked(4L)).thenReturn(List.of()); + Mockito.when(remoteAccessVpnDao.findByPublicIpAddress(4L)).thenReturn(null); + Mockito.when(site2SiteVpnGatewayDao.findByPublicIpAddress(4L)).thenReturn(null); + Mockito.when(networkService.releaseIpAddress(4L)).thenReturn(true); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(worker).getVpcTierKubernetesPublicIp(network); + Mockito.verify(firewallRulesDao).listByIpAndNotRevoked(4L); + Mockito.verify(remoteAccessVpnDao).findByPublicIpAddress(4L); + Mockito.verify(site2SiteVpnGatewayDao).findByPublicIpAddress(4L); + Mockito.verify(networkService).releaseIpAddress(4L); + Mockito.verify(clusterDetailsDao).removeDetail(1L, org.apache.cloudstack.api.ApiConstants.PUBLIC_IP_ID); + } + + @Test + public void managedVpcPublicIpWithUnownedFirewallRuleIsRetained() throws Exception { + prepareVpcPublicIp(); + Mockito.when(firewallRulesDao.listByIpAndNotRevoked(4L)) + .thenReturn(List.of(Mockito.mock(FirewallRuleVO.class))); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(networkService, Mockito.never()).releaseIpAddress(Mockito.anyLong()); + Mockito.verify(clusterDetailsDao, Mockito.never()).removeDetail(Mockito.anyLong(), Mockito.anyString()); + } + + @Test + public void managedVpcPublicIpWithStaticNatBindingIsRetained() throws Exception { + prepareVpcPublicIp(); + publicIp.setOneToOneNat(true); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(networkService, Mockito.never()).releaseIpAddress(Mockito.anyLong()); + Mockito.verify(clusterDetailsDao, Mockito.never()).removeDetail(Mockito.anyLong(), Mockito.anyString()); + } + + @Test + public void managedVpcPublicIpWithRemoteAccessVpnIsRetained() throws Exception { + prepareVpcPublicIp(); + Mockito.when(firewallRulesDao.listByIpAndNotRevoked(4L)).thenReturn(List.of()); + Mockito.when(remoteAccessVpnDao.findByPublicIpAddress(4L)).thenReturn(Mockito.mock(RemoteAccessVpnVO.class)); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(networkService, Mockito.never()).releaseIpAddress(Mockito.anyLong()); + Mockito.verify(remoteAccessVpnDao).findByPublicIpAddress(4L); + } + + @Test + public void managedVpcPublicIpWithSiteToSiteVpnGatewayIsRetained() throws Exception { + prepareVpcPublicIp(); + Mockito.when(firewallRulesDao.listByIpAndNotRevoked(4L)).thenReturn(List.of()); + Mockito.when(site2SiteVpnGatewayDao.findByPublicIpAddress(4L)).thenReturn(Mockito.mock(Site2SiteVpnGatewayVO.class)); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + + Mockito.verify(networkService, Mockito.never()).releaseIpAddress(Mockito.anyLong()); + Mockito.verify(site2SiteVpnGatewayDao).findByPublicIpAddress(4L); + } + + @Test + public void legacyVpcPublicIpIsRetainedWithoutInspectingOrDeletingResources() throws Exception { + prepareVpcPublicIp(); + + worker.releaseVpcTierPublicIpIfNeeded(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + + Mockito.verifyNoInteractions(firewallRulesDao, remoteAccessVpnDao, site2SiteVpnGatewayDao, networkService); + Mockito.verify(clusterDetailsDao, Mockito.never()).removeDetail(Mockito.anyLong(), Mockito.anyString()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidatorTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidatorTest.java new file mode 100644 index 000000000000..382791b8b792 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNetworkRuleOwnershipValidatorTest.java @@ -0,0 +1,829 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.List; +import java.util.Set; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleAdoptionSpec; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleLifecycleState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNetworkACLItemMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.NetworkModel; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVMMapVO; +import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.network.vpc.NetworkACL; +import com.cloud.network.vpc.NetworkACLItem; +import com.cloud.network.vpc.NetworkACLItemDao; +import com.cloud.network.vpc.NetworkACLItemVO; +import com.cloud.network.vpc.NetworkACLVO; +import com.cloud.network.vpc.dao.NetworkACLDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.Nic; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterNetworkRuleOwnershipValidatorTest { + + @Mock + private KubernetesClusterManagerImpl manager; + @Mock + private KubernetesCluster cluster; + @Mock + private KubernetesClusterDao clusterDao; + @Mock + private KubernetesClusterDetailsDao clusterDetailsDao; + @Mock + private KubernetesClusterVmMapDao vmMapDao; + @Mock + private KubernetesClusterFirewallRuleMapDao firewallRuleMapDao; + @Mock + private KubernetesClusterNetworkACLItemMapDao aclItemMapDao; + @Mock + private KubernetesSupportedVersionDao supportedVersionDao; + @Mock + private NetworkDao networkDao; + @Mock + private NetworkACLDao networkACLDao; + @Mock + private NetworkOfferingDao networkOfferingDao; + @Mock + private FirewallRulesDao firewallRulesDao; + @Mock + private PortForwardingRulesDao portForwardingRulesDao; + @Mock + private LoadBalancerDao loadBalancerDao; + @Mock + private LoadBalancerVMMapDao loadBalancerVMMapDao; + @Mock + private NetworkACLItemDao networkACLItemDao; + @Mock + private VMInstanceDao vmInstanceDao; + @Mock + private NetworkModel networkModel; + @Mock + private IpAddress publicIp; + + private KubernetesClusterStartWorker worker; + private KubernetesClusterNetworkRuleOwnershipValidator validator; + + @Before + public void setUp() { + manager.kubernetesClusterDao = clusterDao; + manager.kubernetesClusterDetailsDao = clusterDetailsDao; + manager.kubernetesClusterVmMapDao = vmMapDao; + manager.kubernetesClusterFirewallRuleMapDao = firewallRuleMapDao; + manager.kubernetesClusterNetworkACLItemMapDao = aclItemMapDao; + manager.kubernetesSupportedVersionDao = supportedVersionDao; + worker = Mockito.spy(new KubernetesClusterStartWorker(cluster, manager)); + worker.networkDao = networkDao; + worker.networkACLDao = networkACLDao; + worker.networkOfferingDao = networkOfferingDao; + worker.firewallRulesDao = firewallRulesDao; + worker.portForwardingRulesDao = portForwardingRulesDao; + worker.loadBalancerDao = loadBalancerDao; + worker.loadBalancerVMMapDao = loadBalancerVMMapDao; + worker.networkACLItemDao = networkACLItemDao; + worker.vmInstanceDao = vmInstanceDao; + worker.networkModel = networkModel; + validator = new KubernetesClusterNetworkRuleOwnershipValidator(worker); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getName()).thenReturn("cluster"); + } + + @Test + public void directNetworkHasNoManagedRuleTopology() { + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + + Assert.assertTrue(validator.getExpectedLogicalRoles(network, List.of()).isEmpty()); + } + + @Test + public void nonConserveVpcUsesClusterScopedApiPortForwardingRole() { + NetworkVO network = Mockito.mock(NetworkVO.class); + NetworkOfferingVO offering = Mockito.mock(NetworkOfferingVO.class); + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(1L, 20L, true); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_ALLOW); + Mockito.when(network.getNetworkOfferingId()).thenReturn(30L); + Mockito.when(networkOfferingDao.findById(30L)).thenReturn(offering); + Mockito.when(offering.isConserveMode()).thenReturn(false); + + Set roles = validator.getExpectedLogicalRoles(network, List.of(control)); + + Assert.assertTrue(roles.contains("API_PORT_FORWARD")); + Assert.assertFalse(roles.contains("API_PORT_FORWARD:20")); + Assert.assertTrue(roles.contains("SSH_PORT_FORWARD:20")); + } + + @Test + public void isolatedTopologyIncludesEveryNodeClassWithoutDuplicateRoles() { + NetworkVO network = Mockito.mock(NetworkVO.class); + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(1L, 20L, true); + KubernetesClusterVmMapVO workerNode = new KubernetesClusterVmMapVO(1L, 21L, false); + KubernetesClusterVmMapVO etcd = new KubernetesClusterVmMapVO(1L, 30L, false); + etcd.setEtcdNode(true); + KubernetesClusterVmMapVO external = new KubernetesClusterVmMapVO(1L, 40L, false); + external.setExternalNode(true); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(null); + + Set roles = validator.getExpectedLogicalRoles(network, List.of(control, workerNode, etcd, external)); + + Assert.assertEquals(Set.of("API_FIREWALL", "SSH_FIREWALL", "EXTERNAL_SSH_FIREWALL:40", + "ETCD_SSH_FIREWALL:30", "API_LOAD_BALANCER", "SSH_PORT_FORWARD:20", + "SSH_PORT_FORWARD:21", "SSH_PORT_FORWARD:30", "SSH_PORT_FORWARD:40"), roles); + } + + @Test(expected = InvalidParameterValueException.class) + public void defaultDenyVpcCannotBeAdoptedOrReconciled() { + NetworkVO network = Mockito.mock(NetworkVO.class); + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(1L, 20L, true); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_DENY); + + validator.getExpectedLogicalRoles(network, List.of(control)); + } + + @Test(expected = InvalidParameterValueException.class) + public void nullAclVpcCannotBeAdoptedOrReconciled() { + NetworkVO network = Mockito.mock(NetworkVO.class); + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(1L, 20L, true); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(network.getNetworkACLId()).thenReturn(null); + + validator.getExpectedLogicalRoles(network, List.of(control)); + } + + @Test(expected = InvalidParameterValueException.class) + public void sharedVpcAclCannotBeAdoptedOrReconciled() { + NetworkVO network = Mockito.mock(NetworkVO.class); + NetworkVO otherTier = Mockito.mock(NetworkVO.class); + NetworkACLVO acl = Mockito.mock(NetworkACLVO.class); + KubernetesClusterVmMapVO control = new KubernetesClusterVmMapVO(1L, 20L, true); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(network.getNetworkACLId()).thenReturn(50L); + Mockito.when(networkACLDao.findById(50L)).thenReturn(acl); + Mockito.when(acl.getVpcId()).thenReturn(10L); + Mockito.when(otherTier.getId()).thenReturn(3L); + Mockito.when(networkDao.listByAclId(50L)).thenReturn(List.of(network, otherTier)); + + validator.getExpectedLogicalRoles(network, List.of(control)); + } + + @Test + public void emptyDirectManifestTransitionsLegacyClusterToManaged() { + KubernetesClusterVO locked = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(clusterDao.lockRow(1L, true)).thenReturn(locked); + Mockito.when(locked.getId()).thenReturn(1L); + Mockito.when(locked.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(locked.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(locked.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(locked.getNetworkId()).thenReturn(2L); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.when(vmMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(firewallRuleMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(aclItemMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(clusterDao.update(1L, locked)).thenReturn(true); + + Assert.assertTrue(validator.adopt(List.of())); + + Mockito.verify(locked).setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.verify(clusterDao).update(1L, locked); + Mockito.verifyNoInteractions(networkOfferingDao, networkACLDao); + } + + @Test + public void adoptionPersistsExactIsolatedManifestAcrossFirewallPortForwardingAndLoadBalancerResources() + throws ManagementServerException { + LegacySingleControlTopology topology = mockLegacySingleControlTopology(null, null, false); + FirewallRuleVO apiFirewall = mockFirewallRule(101L, "api-firewall", + KubernetesClusterActionWorker.CLUSTER_API_PORT, KubernetesClusterActionWorker.CLUSTER_API_PORT); + FirewallRuleVO sshFirewall = mockFirewallRule(102L, "ssh-firewall", + KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT, + KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT); + LoadBalancerVO loadBalancer = mockApiLoadBalancer(103L, "api-load-balancer"); + Long controlVmId = topology.vm.getId(); + String controlVmUuid = topology.vm.getUuid(); + String controlIp = topology.nic.getIPv4Address(); + PortForwardingRuleVO portForwardingRule = mockSshPortForwardingRule(104L, "control-ssh", + KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT, controlVmId, controlIp); + LoadBalancerVMMapVO backend = Mockito.mock(LoadBalancerVMMapVO.class); + Mockito.when(backend.getInstanceId()).thenReturn(controlVmId); + Mockito.when(backend.getInstanceIp()).thenReturn(controlIp); + mockFirewallResource(apiFirewall); + mockFirewallResource(sshFirewall); + mockLoadBalancerResource(loadBalancer); + Mockito.when(portForwardingRulesDao.findByUuid(portForwardingRule.getUuid())).thenReturn(portForwardingRule); + Mockito.when(firewallRulesDao.lockRow(portForwardingRule.getId(), true)).thenReturn(portForwardingRule); + Mockito.when(portForwardingRulesDao.findById(portForwardingRule.getId())).thenReturn(portForwardingRule); + Mockito.when(loadBalancerVMMapDao.listByLoadBalancerId(loadBalancer.getId(), false)).thenReturn(List.of(backend)); + Mockito.when(firewallRuleMapDao.persist(Mockito.any(KubernetesClusterFirewallRuleMapVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(clusterDao.update(1L, topology.cluster)).thenReturn(true); + + List specs = List.of( + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.FIREWALL, + KubernetesClusterNetworkRuleRole.API_FIREWALL, apiFirewall.getUuid(), null), + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.FIREWALL, + KubernetesClusterNetworkRuleRole.SSH_FIREWALL, sshFirewall.getUuid(), null), + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.LOAD_BALANCER, + KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER, loadBalancer.getUuid(), null), + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, portForwardingRule.getUuid(), + controlVmUuid)); + + Assert.assertTrue(validator.adopt(specs)); + + verifyPersistedFirewallOwnership(apiFirewall.getId(), "API_FIREWALL"); + verifyPersistedFirewallOwnership(sshFirewall.getId(), "SSH_FIREWALL"); + verifyPersistedFirewallOwnership(loadBalancer.getId(), "API_LOAD_BALANCER"); + verifyPersistedFirewallOwnership(portForwardingRule.getId(), "SSH_PORT_FORWARD:20"); + Mockito.verify(topology.cluster).setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + } + + @Test + public void adoptionPersistsExactDedicatedVpcAclManifest() throws ManagementServerException { + LegacySingleControlTopology topology = mockLegacySingleControlTopology(10L, 50L, false); + NetworkACLVO acl = Mockito.mock(NetworkACLVO.class); + Mockito.when(acl.getVpcId()).thenReturn(10L); + Mockito.when(networkACLDao.findById(50L)).thenReturn(acl); + Mockito.when(networkDao.listByAclId(50L)).thenReturn(List.of(topology.network)); + NetworkACLItemVO apiAcl = mockNetworkAclItem(201L, "api-acl", + KubernetesClusterActionWorker.CLUSTER_API_PORT); + NetworkACLItemVO sshAcl = mockNetworkAclItem(202L, "ssh-acl", + KubernetesClusterActionWorker.DEFAULT_SSH_PORT); + mockNetworkAclItemResource(apiAcl); + mockNetworkAclItemResource(sshAcl); + Mockito.when(portForwardingRulesDao.listByIpAndNotRevoked(100L)).thenReturn(List.of()); + Mockito.when(aclItemMapDao.persist(Mockito.any(KubernetesClusterNetworkACLItemMapVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(clusterDao.update(1L, topology.cluster)).thenReturn(true); + + List specs = List.of( + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.NETWORK_ACL_ITEM, + KubernetesClusterNetworkRuleRole.API_ACL, apiAcl.getUuid(), null), + new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.NETWORK_ACL_ITEM, + KubernetesClusterNetworkRuleRole.SSH_ACL, sshAcl.getUuid(), null)); + + Assert.assertTrue(validator.adopt(specs)); + + verifyPersistedAclOwnership(apiAcl.getId(), "API_ACL"); + verifyPersistedAclOwnership(sshAcl.getId(), "SSH_ACL"); + Mockito.verify(topology.cluster).setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptionRejectsMissingVmMappings() { + KubernetesClusterVO locked = mockLegacyClusterForVmValidation(List.of()); + Mockito.when(locked.getTotalNodeCount()).thenReturn(2L); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptionRejectsMisclassifiedVmMappings() { + KubernetesClusterVmMapVO firstControl = new KubernetesClusterVmMapVO(1L, 20L, true); + KubernetesClusterVmMapVO secondControl = new KubernetesClusterVmMapVO(1L, 21L, true); + KubernetesClusterVO locked = mockLegacyClusterForVmValidation(List.of(firstControl, secondControl)); + Mockito.when(locked.getTotalNodeCount()).thenReturn(2L); + Mockito.when(locked.getControlNodeCount()).thenReturn(1L); + mockOwnedVm(20L); + mockOwnedVm(21L); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptionRejectsOverlappingVmRoles() { + KubernetesClusterVmMapVO conflicting = new KubernetesClusterVmMapVO(1L, 20L, true); + conflicting.setEtcdNode(true); + KubernetesClusterVO locked = mockLegacyClusterForVmValidation(List.of(conflicting)); + Mockito.when(locked.getTotalNodeCount()).thenReturn(1L); + + validator.adopt(List.of()); + } + + @Test + public void adoptionUsesPersistedEtcdSshPortWhenConfigurationChanged() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(true, 51000); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(topology.rule.getId(), true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(topology.rule.getId())).thenReturn(topology.rule); + Mockito.when(firewallRuleMapDao.persist(Mockito.any(KubernetesClusterFirewallRuleMapVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(clusterDao.update(1L, topology.cluster)).thenReturn(true); + + Assert.assertTrue(validator.adopt(List.of(spec))); + + Mockito.verify(firewallRuleMapDao).persist(Mockito.argThat(mapping -> + mapping.getFirewallRuleId() == topology.rule.getId() + && "SSH_PORT_FORWARD:30".equals(mapping.getLogicalRole()))); + Mockito.verify(topology.cluster).setNetworkRuleOwnershipState(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + } + + @Test(expected = InvalidParameterValueException.class) + public void omittedEtcdSshRuleIsDetectedEvenWhenConfigurationChanged() throws ManagementServerException { + mockLegacyDynamicSshVpcTopology(true, 51000); + + validator.adopt(List.of()); + } + + @Test + public void adoptionUsesPersistedExternalSshPortAfterPortHistoryDevelopsGap() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(topology.rule.getId(), true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(topology.rule.getId())).thenReturn(topology.rule); + Mockito.when(firewallRuleMapDao.persist(Mockito.any(KubernetesClusterFirewallRuleMapVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(clusterDao.update(1L, topology.cluster)).thenReturn(true); + + Assert.assertTrue(validator.adopt(List.of(spec))); + + Mockito.verify(firewallRuleMapDao).persist(Mockito.argThat(mapping -> + mapping.getFirewallRuleId() == topology.rule.getId() + && "SSH_PORT_FORWARD:30".equals(mapping.getLogicalRole()))); + } + + @Test(expected = InvalidParameterValueException.class) + public void adoptionRejectsResourceOwnedByAnotherCluster() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + KubernetesClusterFirewallRuleMapVO ownership = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(topology.rule.getId(), true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(topology.rule.getId())).thenReturn(topology.rule); + Mockito.when(ownership.getClusterId()).thenReturn(2L); + Mockito.when(ownership.getLogicalRole()).thenReturn("SSH_PORT_FORWARD:30"); + Mockito.when(firewallRuleMapDao.findByFirewallRuleId(topology.rule.getId())).thenReturn(ownership); + + validator.adopt(List.of(spec)); + } + + @Test + public void managedAdoptionAcceptsOnlyExactIdempotentReplay() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + KubernetesClusterFirewallRuleMapVO ownership = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Long ruleId = topology.rule.getId(); + Mockito.when(topology.cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(ruleId, true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(ruleId)).thenReturn(topology.rule); + Mockito.when(ownership.getClusterId()).thenReturn(1L); + Mockito.when(ownership.getLogicalRole()).thenReturn("SSH_PORT_FORWARD:30"); + Mockito.when(ownership.getFirewallRuleId()).thenReturn(ruleId); + Mockito.when(ownership.getLifecycleState()).thenReturn(KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(firewallRuleMapDao.findByFirewallRuleId(ruleId)).thenReturn(ownership); + Mockito.when(firewallRuleMapDao.findByClusterIdAndLogicalRole(1L, "SSH_PORT_FORWARD:30")).thenReturn(ownership); + Mockito.when(firewallRuleMapDao.listByClusterId(1L)).thenReturn(List.of(ownership)); + + Assert.assertTrue(validator.adopt(List.of(spec))); + + Mockito.verify(firewallRuleMapDao, Mockito.never()).persist(Mockito.any()); + Mockito.verify(clusterDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + + @Test(expected = InvalidParameterValueException.class) + public void managedAdoptionRejectsNonExactReplay() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + Long ruleId = topology.rule.getId(); + Mockito.when(topology.cluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(ruleId, true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(ruleId)).thenReturn(topology.rule); + + try { + validator.adopt(List.of(spec)); + } finally { + Mockito.verify(firewallRuleMapDao, Mockito.never()).persist(Mockito.any()); + Mockito.verify(aclItemMapDao, Mockito.never()).persist(Mockito.any()); + Mockito.verify(clusterDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + } + + @Test(expected = CloudRuntimeException.class) + public void adoptionDoesNotUpdateClusterWhenOwnershipPersistenceFails() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + KubernetesClusterNetworkRuleAdoptionSpec spec = new KubernetesClusterNetworkRuleAdoptionSpec( + KubernetesClusterNetworkRuleRole.ResourceType.PORT_FORWARDING, + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD, topology.rule.getUuid(), topology.vm.getUuid()); + Mockito.when(portForwardingRulesDao.findByUuid(topology.rule.getUuid())).thenReturn(topology.rule); + Mockito.when(firewallRulesDao.lockRow(topology.rule.getId(), true)).thenReturn(topology.rule); + Mockito.when(portForwardingRulesDao.findById(topology.rule.getId())).thenReturn(topology.rule); + Mockito.when(firewallRuleMapDao.persist(Mockito.any(KubernetesClusterFirewallRuleMapVO.class))).thenReturn(null); + + try { + validator.adopt(List.of(spec)); + } finally { + Mockito.verify(clusterDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void omittedExternalSshRuleIsDetectedAfterPortHistoryDevelopsGap() throws ManagementServerException { + mockLegacyDynamicSshVpcTopology(false, 2226); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void multipleExternalSshRulesTargetingSameVmAreRejected() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + PortForwardingRuleVO duplicate = mockSshPortForwardingRule(201L, "external-rule-duplicate", 2227); + Mockito.when(portForwardingRulesDao.listByIpAndNotRevoked(100L)).thenReturn(List.of(topology.rule, duplicate)); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void stagedExternalSshRuleCannotBeOmittedFromAdoption() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + Mockito.when(topology.rule.getState()).thenReturn(FirewallRule.State.Add); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void staleDestinationExternalSshRuleCannotBeOmittedFromAdoption() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + Mockito.when(topology.rule.getDestinationIpAddress()).thenReturn(new Ip("10.0.0.99")); + Assert.assertEquals("10.0.0.99", topology.rule.getDestinationIpAddress().addr()); + + validator.adopt(List.of()); + } + + @Test(expected = InvalidParameterValueException.class) + public void activeAndStagedExternalSshRulesTargetingSameVmAreRejected() throws ManagementServerException { + LegacyDynamicSshTopology topology = mockLegacyDynamicSshVpcTopology(false, 2226); + PortForwardingRuleVO stagedDuplicate = mockSshPortForwardingRule(201L, "external-rule-staged", 2227); + Mockito.when(stagedDuplicate.getState()).thenReturn(FirewallRule.State.Add); + Mockito.when(portForwardingRulesDao.listByIpAndNotRevoked(100L)).thenReturn(List.of(topology.rule, stagedDuplicate)); + + validator.adopt(List.of()); + } + + private LegacySingleControlTopology mockLegacySingleControlTopology(Long vpcId, Long networkAclId, + boolean conserveMode) throws ManagementServerException { + KubernetesClusterVO locked = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + KubernetesClusterVmMapVO controlMap = new KubernetesClusterVmMapVO(1L, 20L, true); + VMInstanceVO controlVm = Mockito.mock(VMInstanceVO.class); + Nic controlNic = Mockito.mock(Nic.class); + + Mockito.when(clusterDao.lockRow(1L, true)).thenReturn(locked); + Mockito.when(locked.getId()).thenReturn(1L); + Mockito.when(locked.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(locked.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(locked.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(locked.getNetworkId()).thenReturn(2L); + Mockito.when(locked.getAccountId()).thenReturn(3L); + Mockito.when(locked.getDomainId()).thenReturn(4L); + Mockito.when(locked.getControlNodeCount()).thenReturn(1L); + Mockito.when(locked.getNodeCount()).thenReturn(0L); + Mockito.when(locked.getEtcdNodeCount()).thenReturn(0L); + Mockito.when(locked.getTotalNodeCount()).thenReturn(1L); + Mockito.when(cluster.getAccountId()).thenReturn(3L); + Mockito.when(cluster.getDomainId()).thenReturn(4L); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(network.getVpcId()).thenReturn(vpcId); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + if (vpcId != null) { + NetworkOfferingVO offering = Mockito.mock(NetworkOfferingVO.class); + Mockito.when(network.getNetworkACLId()).thenReturn(networkAclId); + Mockito.when(network.getNetworkOfferingId()).thenReturn(40L); + Mockito.when(networkOfferingDao.findById(40L)).thenReturn(offering); + Mockito.when(offering.isConserveMode()).thenReturn(conserveMode); + } + Mockito.when(vmMapDao.listByClusterId(1L)).thenReturn(List.of(controlMap)); + Mockito.when(controlVm.getAccountId()).thenReturn(3L); + Mockito.when(controlVm.getDomainId()).thenReturn(4L); + Mockito.when(controlVm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(20L)).thenReturn(controlVm); + if (vpcId == null) { + Mockito.when(controlVm.getId()).thenReturn(20L); + Mockito.when(controlVm.getUuid()).thenReturn("control-vm"); + Mockito.when(vmInstanceDao.findByUuid("control-vm")).thenReturn(controlVm); + Mockito.when(controlNic.getIPv4Address()).thenReturn("10.0.0.20"); + Mockito.when(networkModel.getNicInNetwork(20L, 2L)).thenReturn(controlNic); + } + Mockito.doReturn(publicIp).when(worker).getPublicIp(network); + Mockito.when(publicIp.getId()).thenReturn(100L); + Mockito.when(publicIp.getAllocatedToAccountId()).thenReturn(3L); + Mockito.when(publicIp.getAllocatedInDomainId()).thenReturn(4L); + Mockito.when(publicIp.getAssociatedWithNetworkId()).thenReturn(2L); + Mockito.when(firewallRuleMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(aclItemMapDao.listByClusterId(1L)).thenReturn(List.of()); + return new LegacySingleControlTopology(locked, network, controlVm, controlNic); + } + + private FirewallRuleVO mockFirewallRule(long id, String uuid, int startPort, int endPort) { + FirewallRuleVO rule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(rule.getId()).thenReturn(id); + Mockito.when(rule.getUuid()).thenReturn(uuid); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(rule.getPurpose()).thenReturn(FirewallRule.Purpose.Firewall); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(100L); + Mockito.when(rule.getNetworkId()).thenReturn(2L); + Mockito.when(rule.getAccountId()).thenReturn(3L); + Mockito.when(rule.getDomainId()).thenReturn(4L); + Mockito.when(rule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(rule.getSourcePortStart()).thenReturn(startPort); + Mockito.when(rule.getSourcePortEnd()).thenReturn(endPort); + Mockito.when(rule.getTrafficType()).thenReturn(FirewallRule.TrafficType.Ingress); + Mockito.when(rule.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS)); + return rule; + } + + private LoadBalancerVO mockApiLoadBalancer(long id, String uuid) { + LoadBalancerVO rule = Mockito.mock(LoadBalancerVO.class); + Mockito.when(rule.getId()).thenReturn(id); + Mockito.when(rule.getUuid()).thenReturn(uuid); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(rule.getPurpose()).thenReturn(FirewallRule.Purpose.LoadBalancing); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(100L); + Mockito.when(rule.getNetworkId()).thenReturn(2L); + Mockito.when(rule.getAccountId()).thenReturn(3L); + Mockito.when(rule.getDomainId()).thenReturn(4L); + Mockito.when(rule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(rule.getSourcePortStart()).thenReturn(KubernetesClusterActionWorker.CLUSTER_API_PORT); + Mockito.when(rule.getSourcePortEnd()).thenReturn(KubernetesClusterActionWorker.CLUSTER_API_PORT); + Mockito.when(rule.getDefaultPortStart()).thenReturn(KubernetesClusterActionWorker.CLUSTER_API_PORT); + Mockito.when(rule.getDefaultPortEnd()).thenReturn(KubernetesClusterActionWorker.CLUSTER_API_PORT); + Mockito.when(rule.getName()).thenReturn("api-lb"); + Mockito.when(rule.getAlgorithm()).thenReturn("roundrobin"); + Mockito.when(rule.getLbProtocol()).thenReturn(NetUtils.TCP_PROTO); + return rule; + } + + private void mockFirewallResource(FirewallRuleVO rule) { + Mockito.when(firewallRulesDao.findByUuid(rule.getUuid())).thenReturn(rule); + Mockito.when(firewallRulesDao.lockRow(rule.getId(), true)).thenReturn(rule); + Mockito.when(firewallRulesDao.findById(rule.getId())).thenReturn(rule); + } + + private void mockLoadBalancerResource(LoadBalancerVO rule) { + Mockito.when(loadBalancerDao.findByUuid(rule.getUuid())).thenReturn(rule); + Mockito.when(firewallRulesDao.lockRow(rule.getId(), true)).thenReturn(rule); + Mockito.when(loadBalancerDao.findById(rule.getId())).thenReturn(rule); + } + + private NetworkACLItemVO mockNetworkAclItem(long id, String uuid, int port) { + NetworkACLItemVO item = Mockito.mock(NetworkACLItemVO.class); + Mockito.when(item.getId()).thenReturn(id); + Mockito.when(item.getUuid()).thenReturn(uuid); + Mockito.when(item.getAclId()).thenReturn(50L); + Mockito.when(item.getState()).thenReturn(NetworkACLItem.State.Active); + Mockito.when(item.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(item.getSourcePortStart()).thenReturn(port); + Mockito.when(item.getSourcePortEnd()).thenReturn(port); + Mockito.when(item.getTrafficType()).thenReturn(NetworkACLItem.TrafficType.Ingress); + Mockito.when(item.getAction()).thenReturn(NetworkACLItem.Action.Allow); + Mockito.when(item.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS, NetUtils.ALL_IP6_CIDRS)); + return item; + } + + private void mockNetworkAclItemResource(NetworkACLItemVO item) { + Mockito.when(networkACLItemDao.findByUuid(item.getUuid())).thenReturn(item); + Mockito.when(networkACLItemDao.lockRow(item.getId(), true)).thenReturn(item); + Mockito.when(networkACLItemDao.findById(item.getId())).thenReturn(item); + } + + private void verifyPersistedFirewallOwnership(long resourceId, String logicalRole) { + Mockito.verify(firewallRuleMapDao).persist(Mockito.argThat(mapping -> + mapping.getFirewallRuleId() == resourceId + && logicalRole.equals(mapping.getLogicalRole()) + && KubernetesClusterNetworkRuleLifecycleState.ACTIVE.equals(mapping.getLifecycleState()))); + } + + private void verifyPersistedAclOwnership(long resourceId, String logicalRole) { + Mockito.verify(aclItemMapDao).persist(Mockito.argThat(mapping -> + mapping.getNetworkAclItemId() == resourceId + && logicalRole.equals(mapping.getLogicalRole()) + && KubernetesClusterNetworkRuleLifecycleState.ACTIVE.equals(mapping.getLifecycleState()))); + } + + private LegacyDynamicSshTopology mockLegacyDynamicSshVpcTopology(boolean etcdNode, int sourcePort) + throws ManagementServerException { + KubernetesClusterVO locked = Mockito.mock(KubernetesClusterVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + NetworkOfferingVO offering = Mockito.mock(NetworkOfferingVO.class); + KubernetesClusterVmMapVO controlMap = new KubernetesClusterVmMapVO(1L, 20L, true); + KubernetesClusterVmMapVO dynamicMap = new KubernetesClusterVmMapVO(1L, 30L, false); + dynamicMap.setEtcdNode(etcdNode); + dynamicMap.setExternalNode(!etcdNode); + VMInstanceVO controlVm = Mockito.mock(VMInstanceVO.class); + VMInstanceVO dynamicVm = Mockito.mock(VMInstanceVO.class); + PortForwardingRuleVO dynamicRule = mockSshPortForwardingRule(200L, "dynamic-rule", sourcePort); + Nic dynamicNic = Mockito.mock(Nic.class); + + Mockito.when(clusterDao.lockRow(1L, true)).thenReturn(locked); + Mockito.when(locked.getId()).thenReturn(1L); + Mockito.when(locked.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(locked.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(locked.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(locked.getNetworkId()).thenReturn(2L); + Mockito.when(locked.getAccountId()).thenReturn(3L); + Mockito.when(locked.getDomainId()).thenReturn(4L); + Mockito.when(locked.getControlNodeCount()).thenReturn(1L); + Mockito.when(locked.getNodeCount()).thenReturn(etcdNode ? 0L : 1L); + Mockito.when(locked.getEtcdNodeCount()).thenReturn(etcdNode ? 1L : 0L); + Mockito.when(locked.getTotalNodeCount()).thenReturn(2L); + Mockito.when(cluster.getAccountId()).thenReturn(3L); + Mockito.when(cluster.getDomainId()).thenReturn(4L); + Mockito.when(networkDao.findById(2L)).thenReturn(network); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_ALLOW); + Mockito.when(network.getNetworkOfferingId()).thenReturn(40L); + Mockito.when(networkOfferingDao.findById(40L)).thenReturn(offering); + Mockito.when(offering.isConserveMode()).thenReturn(false); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(vmMapDao.listByClusterId(1L)).thenReturn(List.of(controlMap, dynamicMap)); + + Mockito.when(controlVm.getAccountId()).thenReturn(3L); + Mockito.when(controlVm.getDomainId()).thenReturn(4L); + Mockito.when(controlVm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(dynamicVm.getId()).thenReturn(30L); + Mockito.when(dynamicVm.getUuid()).thenReturn("dynamic-vm"); + Mockito.when(dynamicVm.getAccountId()).thenReturn(3L); + Mockito.when(dynamicVm.getDomainId()).thenReturn(4L); + Mockito.when(dynamicVm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(20L)).thenReturn(controlVm); + Mockito.when(vmInstanceDao.findById(30L)).thenReturn(dynamicVm); + Mockito.when(vmInstanceDao.findByUuid("dynamic-vm")).thenReturn(dynamicVm); + + Mockito.doReturn(publicIp).when(worker).getPublicIp(network); + Mockito.when(publicIp.getId()).thenReturn(100L); + Mockito.when(publicIp.getAllocatedToAccountId()).thenReturn(3L); + Mockito.when(publicIp.getAllocatedInDomainId()).thenReturn(4L); + Mockito.when(publicIp.getAssociatedWithNetworkId()).thenReturn(2L); + + Mockito.when(portForwardingRulesDao.listByIpAndNotRevoked(100L)).thenReturn(List.of(dynamicRule)); + Mockito.when(dynamicNic.getIPv4Address()).thenReturn("10.0.0.30"); + Mockito.when(networkModel.getNicInNetwork(30L, 2L)).thenReturn(dynamicNic); + Mockito.when(firewallRuleMapDao.listByClusterId(1L)).thenReturn(List.of()); + Mockito.when(aclItemMapDao.listByClusterId(1L)).thenReturn(List.of()); + + return new LegacyDynamicSshTopology(locked, dynamicVm, dynamicRule); + } + + private KubernetesClusterVO mockLegacyClusterForVmValidation(List vmMaps) { + KubernetesClusterVO locked = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(clusterDao.lockRow(1L, true)).thenReturn(locked); + Mockito.when(locked.getId()).thenReturn(1L); + Mockito.when(locked.getName()).thenReturn("cluster"); + Mockito.when(locked.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(locked.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(locked.getNetworkRuleOwnershipState()).thenReturn( + KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(locked.getNetworkId()).thenReturn(2L); + Mockito.when(locked.getAccountId()).thenReturn(3L); + Mockito.when(locked.getDomainId()).thenReturn(4L); + Mockito.when(networkDao.findById(2L)).thenReturn(Mockito.mock(NetworkVO.class)); + Mockito.when(vmMapDao.listByClusterId(1L)).thenReturn(vmMaps); + return locked; + } + + private void mockOwnedVm(long vmId) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getAccountId()).thenReturn(3L); + Mockito.when(vm.getDomainId()).thenReturn(4L); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(vmId)).thenReturn(vm); + } + + private PortForwardingRuleVO mockSshPortForwardingRule(long id, String uuid, int sourcePort) { + return mockSshPortForwardingRule(id, uuid, sourcePort, 30L, "10.0.0.30"); + } + + private PortForwardingRuleVO mockSshPortForwardingRule(long id, String uuid, int sourcePort, + long vmId, String destinationIp) { + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(rule.getId()).thenReturn(id); + Mockito.when(rule.getUuid()).thenReturn(uuid); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(rule.getPurpose()).thenReturn(FirewallRule.Purpose.PortForwarding); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(100L); + Mockito.when(rule.getNetworkId()).thenReturn(2L); + Mockito.when(rule.getAccountId()).thenReturn(3L); + Mockito.when(rule.getDomainId()).thenReturn(4L); + Mockito.when(rule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(rule.getSourcePortStart()).thenReturn(sourcePort); + Mockito.when(rule.getSourcePortEnd()).thenReturn(sourcePort); + Mockito.when(rule.getDestinationPortStart()).thenReturn(22); + Mockito.when(rule.getDestinationPortEnd()).thenReturn(22); + Mockito.when(rule.getVirtualMachineId()).thenReturn(vmId); + Mockito.when(rule.getDestinationIpAddress()).thenReturn(new Ip(destinationIp)); + return rule; + } + + private static final class LegacySingleControlTopology { + private final KubernetesClusterVO cluster; + private final NetworkVO network; + private final VMInstanceVO vm; + private final Nic nic; + + private LegacySingleControlTopology(KubernetesClusterVO cluster, NetworkVO network, VMInstanceVO vm, + Nic nic) { + this.cluster = cluster; + this.network = network; + this.vm = vm; + this.nic = nic; + } + } + + private static final class LegacyDynamicSshTopology { + private final KubernetesClusterVO cluster; + private final VMInstanceVO vm; + private final PortForwardingRuleVO rule; + + private LegacyDynamicSshTopology(KubernetesClusterVO cluster, VMInstanceVO vm, + PortForwardingRuleVO rule) { + this.cluster = cluster; + this.vm = vm; + this.rule = rule; + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorkerTest.java new file mode 100644 index 000000000000..6d2ebb06fe13 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorkerTest.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; +import com.cloud.vm.UserVmVO; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterRemoveWorkerTest { + + @Mock + private KubernetesCluster cluster; + @Mock + private KubernetesClusterManagerImpl manager; + @Mock + private KubernetesClusterDao clusterDao; + @Mock + private KubernetesClusterFirewallRuleMapDao firewallRuleMapDao; + @Mock + private PortForwardingRulesDao portForwardingRulesDao; + @Mock + private NetworkModel networkModel; + @Mock + private Network network; + @Mock + private IpAddress publicIp; + @Mock + private UserVmVO vm; + + private KubernetesClusterRemoveWorker worker; + + @Before + public void setUp() { + manager.kubernetesClusterDao = clusterDao; + manager.kubernetesClusterFirewallRuleMapDao = firewallRuleMapDao; + worker = Mockito.spy(new KubernetesClusterRemoveWorker(cluster, manager)); + worker.portForwardingRulesDao = portForwardingRulesDao; + worker.networkModel = networkModel; + Mockito.when(cluster.getId()).thenReturn(1L); + } + + @Test + public void directRemovedNodeUsesClusterNicWithoutPortForwarding() throws Exception { + Nic nic = Mockito.mock(Nic.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(networkModel.getNicInNetwork(3L, 2L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("192.0.2.10"); + + Pair endpoint = worker.getRemovedNodeIpSshPort(3L, network, null); + + Assert.assertEquals("192.0.2.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(KubernetesClusterActionWorker.DEFAULT_SSH_PORT), endpoint.second()); + Mockito.verifyNoInteractions(firewallRuleMapDao, portForwardingRulesDao); + } + + @Test + public void nonDirectRemovedNodeUsesItsOwnedPortForwardingRule() throws Exception { + KubernetesClusterFirewallRuleMapVO mapping = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getId()).thenReturn(2L); + Mockito.when(publicIp.getId()).thenReturn(4L); + Mockito.when(publicIp.getAddress()).thenReturn(new Ip("203.0.113.10")); + Mockito.when(firewallRuleMapDao.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 3L)).thenReturn(mapping); + Mockito.when(mapping.getFirewallRuleId()).thenReturn(5L); + Mockito.when(portForwardingRulesDao.findById(5L)).thenReturn(rule); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(4L); + Mockito.when(rule.getNetworkId()).thenReturn(2L); + Mockito.when(rule.getDestinationPortStart()).thenReturn(KubernetesClusterActionWorker.DEFAULT_SSH_PORT); + Mockito.when(rule.getVirtualMachineId()).thenReturn(3L); + Mockito.when(rule.getSourcePortStart()).thenReturn(2222); + + Pair endpoint = worker.getRemovedNodeIpSshPort(3L, network, publicIp); + + Assert.assertEquals("203.0.113.10", endpoint.first()); + Assert.assertEquals(Integer.valueOf(2222), endpoint.second()); + } + + @Test + public void legacyDirectRemovalNeverDeletesUnownedRules() { + KubernetesClusterVO persisted = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.when(clusterDao.findById(1L)).thenReturn(persisted); + Mockito.when(persisted.getNetworkRuleOwnershipState()) + .thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + + Assert.assertTrue(worker.removeNodePortForwardingRules(3L, network, vm)); + + Mockito.verifyNoInteractions(firewallRuleMapDao); + } + + @Test + public void managedDirectRemovalDeletesOnlyExplicitMappedLeftovers() throws Exception { + KubernetesClusterVO persisted = Mockito.mock(KubernetesClusterVO.class); + KubernetesClusterFirewallRuleMapVO portForwarding = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + KubernetesClusterFirewallRuleMapVO firewall = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.when(network.getVpcId()).thenReturn(null); + Mockito.when(clusterDao.findById(1L)).thenReturn(persisted); + Mockito.when(persisted.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.when(firewallRuleMapDao.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 3L)).thenReturn(portForwarding); + Mockito.when(firewallRuleMapDao.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.EXTERNAL_SSH_FIREWALL_ROLE_PREFIX + 3L)).thenReturn(firewall); + Mockito.doNothing().when(worker).deleteManagedPortForwardingRule(portForwarding); + Mockito.doNothing().when(worker).deleteManagedFirewallRule(firewall); + + Assert.assertTrue(worker.removeNodePortForwardingRules(3L, network, vm)); + + Mockito.verify(worker).deleteManagedPortForwardingRule(portForwarding); + Mockito.verify(worker).deleteManagedFirewallRule(firewall); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java index c220a3468afb..7086c5b2eabb 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java @@ -17,12 +17,52 @@ package com.cloud.kubernetes.cluster.actionworkers; +import java.util.List; +import java.util.Set; + +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterFirewallRuleMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkACLItemMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleLifecycleState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterFirewallRuleMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNetworkACLItemMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.firewall.FirewallService; +import com.cloud.network.lb.LoadBalancingRulesService; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.network.vpc.NetworkACLItem; +import com.cloud.network.vpc.NetworkACLItemDao; +import com.cloud.network.vpc.NetworkACLItemVO; +import com.cloud.network.vpc.NetworkACLService; +import com.cloud.network.vpc.NetworkACL; +import com.cloud.network.vpc.NetworkACLVO; +import com.cloud.network.vpc.dao.NetworkACLDao; +import com.cloud.user.Account; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.Nic; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,6 +73,19 @@ @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterResourceModifierActionWorkerTest { + private static class TestKubernetesClusterResourceModifierActionWorker extends KubernetesClusterResourceModifierActionWorker { + private int provisionAclRuleCalls; + + TestKubernetesClusterResourceModifierActionWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + @Override + protected void provisionVpcTierAllowPortACLRule(Network network, int startPort, int endPort, String logicalRole) { + provisionAclRuleCalls++; + } + } + @Mock private KubernetesClusterDao kubernetesClusterDaoMock; @@ -51,16 +104,78 @@ public class KubernetesClusterResourceModifierActionWorkerTest { @Mock private KubernetesCluster kubernetesClusterMock; + @Mock + private KubernetesClusterVO lockedKubernetesClusterMock; + + @Mock + private KubernetesClusterFirewallRuleMapDao kubernetesClusterFirewallRuleMapDaoMock; + + @Mock + private KubernetesClusterNetworkACLItemMapDao kubernetesClusterNetworkACLItemMapDaoMock; + + @Mock + private FirewallRulesDao firewallRulesDaoMock; + + @Mock + private FirewallService firewallServiceMock; + + @Mock + private PortForwardingRulesDao portForwardingRulesDaoMock; + + @Mock + private RulesService rulesServiceMock; + + @Mock + private NetworkACLItemDao networkACLItemDaoMock; + + @Mock + private NetworkACLService networkACLServiceMock; + + @Mock + private NetworkDao networkDaoMock; + @Mock + private NetworkACLDao networkACLDaoMock; + + @Mock + private LoadBalancerDao loadBalancerDaoMock; + + @Mock + private LoadBalancerVMMapDao loadBalancerVMMapDaoMock; + + @Mock + private LoadBalancingRulesService loadBalancingRulesServiceMock; + private KubernetesClusterResourceModifierActionWorker kubernetesClusterResourceModifierActionWorker; @Before - public void setUp() { + public void setUp() throws Exception { kubernetesClusterManagerMock.kubernetesClusterDao = kubernetesClusterDaoMock; kubernetesClusterManagerMock.kubernetesSupportedVersionDao = kubernetesSupportedVersionDaoMock; kubernetesClusterManagerMock.kubernetesClusterDetailsDao = kubernetesClusterDetailsDaoMock; kubernetesClusterManagerMock.kubernetesClusterVmMapDao = kubernetesClusterVmMapDaoMock; + kubernetesClusterManagerMock.kubernetesClusterFirewallRuleMapDao = kubernetesClusterFirewallRuleMapDaoMock; + kubernetesClusterManagerMock.kubernetesClusterNetworkACLItemMapDao = kubernetesClusterNetworkACLItemMapDaoMock; + + Mockito.when(kubernetesClusterMock.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterDaoMock.lockRow(1L, true)).thenReturn(lockedKubernetesClusterMock); + Mockito.when(lockedKubernetesClusterMock.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); kubernetesClusterResourceModifierActionWorker = new KubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + kubernetesClusterResourceModifierActionWorker.firewallRulesDao = firewallRulesDaoMock; + kubernetesClusterResourceModifierActionWorker.firewallService = firewallServiceMock; + kubernetesClusterResourceModifierActionWorker.portForwardingRulesDao = portForwardingRulesDaoMock; + kubernetesClusterResourceModifierActionWorker.rulesService = rulesServiceMock; + kubernetesClusterResourceModifierActionWorker.networkACLItemDao = networkACLItemDaoMock; + kubernetesClusterResourceModifierActionWorker.networkACLService = networkACLServiceMock; + kubernetesClusterResourceModifierActionWorker.networkDao = networkDaoMock; + kubernetesClusterResourceModifierActionWorker.networkACLDao = networkACLDaoMock; + kubernetesClusterResourceModifierActionWorker.loadBalancerDao = loadBalancerDaoMock; + kubernetesClusterResourceModifierActionWorker.loadBalancerVMMapDao = loadBalancerVMMapDaoMock; + kubernetesClusterResourceModifierActionWorker.lbService = loadBalancingRulesServiceMock; + + Mockito.when(firewallServiceMock.applyIngressFwRules(Mockito.anyLong(), Mockito.any())).thenReturn(true); + Mockito.when(rulesServiceMock.applyPortForwardingRules(Mockito.anyLong(), Mockito.any())).thenReturn(true); + Mockito.when(networkACLServiceMock.applyNetworkACL(Mockito.anyLong())).thenReturn(true); } @Test @@ -135,4 +250,488 @@ public void getKubernetesClusterNodeNamePrefixTestNormalizedPrefixShouldNotStart Mockito.when(kubernetesClusterMock.getName()).thenReturn(originalPrefix); Assert.assertEquals(expectedPrefix, kubernetesClusterResourceModifierActionWorker.getKubernetesClusterNodeNamePrefix()); } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionFirewallRulesRejectsUnownedLegacyRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Account account = Mockito.mock(Account.class); + FirewallRuleVO existingRule = Mockito.mock(FirewallRuleVO.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(existingRule.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS)); + Mockito.when(firewallRulesDaoMock.listByIpPurposePortsProtocolAndNotRevoked(10L, 6443, 6443, + NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall)).thenReturn(List.of(existingRule)); + + kubernetesClusterResourceModifierActionWorker.provisionFirewallRules(publicIp, account, 6443, 6443, + KubernetesClusterActionWorker.API_FIREWALL_ROLE); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionPortForwardingRuleRejectsUnownedLegacyRule() throws Exception { + long publicIpId = 10L; + long networkId = 20L; + long accountId = 30L; + long vmId = 40L; + int sourcePort = 2222; + int destinationPort = 22; + Ip vmIp = new Ip("10.1.1.10"); + IpAddress publicIp = Mockito.mock(IpAddress.class); + com.cloud.network.dao.NetworkVO network = Mockito.mock(com.cloud.network.dao.NetworkVO.class); + Account account = Mockito.mock(Account.class); + Nic nic = Mockito.mock(Nic.class); + PortForwardingRuleVO existingRule = Mockito.mock(PortForwardingRuleVO.class); + kubernetesClusterResourceModifierActionWorker.networkModel = Mockito.mock(NetworkModel.class); + Mockito.when(publicIp.getId()).thenReturn(publicIpId); + Mockito.when(network.getId()).thenReturn(networkId); + Mockito.when(account.getId()).thenReturn(accountId); + Mockito.when(kubernetesClusterResourceModifierActionWorker.networkModel.getNicInNetwork(vmId, networkId)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn(vmIp.addr()); + Mockito.when(portForwardingRulesDaoMock.listByIpAndNotRevoked(publicIpId)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(sourcePort); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(sourcePort); + Mockito.when(existingRule.getDestinationPortStart()).thenReturn(destinationPort); + Mockito.when(existingRule.getDestinationPortEnd()).thenReturn(destinationPort); + Mockito.when(existingRule.getVirtualMachineId()).thenReturn(vmId); + Mockito.when(existingRule.getNetworkId()).thenReturn(networkId); + Mockito.when(existingRule.getAccountId()).thenReturn(accountId); + Mockito.when(existingRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(existingRule.getDestinationIpAddress()).thenReturn(vmIp); + + kubernetesClusterResourceModifierActionWorker.provisionPublicIpPortForwardingRule(publicIp, network, account, + vmId, sourcePort, destinationPort); + + Mockito.verify(rulesServiceMock, Mockito.never()).applyPortForwardingRules(publicIpId, account); + Mockito.verify(portForwardingRulesDaoMock, Mockito.never()).persist(Mockito.any()); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionPortForwardingRuleRejectsAnOverlappingRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Network network = Mockito.mock(Network.class); + Account account = Mockito.mock(Account.class); + Nic nic = Mockito.mock(Nic.class); + PortForwardingRuleVO existingRule = Mockito.mock(PortForwardingRuleVO.class); + kubernetesClusterResourceModifierActionWorker.networkModel = Mockito.mock(NetworkModel.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(kubernetesClusterResourceModifierActionWorker.networkModel.getNicInNetwork(40L, 20L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("10.1.1.10"); + Mockito.when(portForwardingRulesDaoMock.listByIpAndNotRevoked(10L)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(2222); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(2222); + Mockito.when(existingRule.getDestinationPortStart()).thenReturn(80); + + kubernetesClusterResourceModifierActionWorker.provisionPublicIpPortForwardingRule(publicIp, network, account, + 40L, 2222, 22); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionVpcTierAclRuleRejectsUnownedLegacyRule() throws Exception { + com.cloud.network.dao.NetworkVO network = Mockito.mock(com.cloud.network.dao.NetworkVO.class); + NetworkACLItemVO existingRule = Mockito.mock(NetworkACLItemVO.class); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(network.getNetworkACLId()).thenReturn(50L); + Mockito.when(networkDaoMock.findById(20L)).thenReturn(network); + Mockito.when(networkACLItemDaoMock.listByACL(50L)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getState()).thenReturn(NetworkACLItem.State.Active); + Mockito.when(existingRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(6443); + Mockito.when(existingRule.getTrafficType()).thenReturn(NetworkACLItem.TrafficType.Ingress); + Mockito.when(existingRule.getAction()).thenReturn(NetworkACLItem.Action.Allow); + Mockito.when(existingRule.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS, NetUtils.ALL_IP6_CIDRS)); + + kubernetesClusterResourceModifierActionWorker.provisionVpcTierAllowPortACLRule(network, 6443, 6443, + KubernetesClusterActionWorker.API_ACL_ROLE); + } + + @Test(expected = ManagementServerException.class) + public void createVpcTierAclRulesWithoutAclFails() throws Exception { + Network network = Mockito.mock(Network.class); + Mockito.when(network.getNetworkACLId()).thenReturn(null); + TestKubernetesClusterResourceModifierActionWorker worker = + new TestKubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + + worker.createVpcTierAclRules(network); + } + + @Test + public void createVpcTierAclRulesWithDefaultAllowDoesNotProvisionRules() throws Exception { + Network network = Mockito.mock(Network.class); + Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_ALLOW); + TestKubernetesClusterResourceModifierActionWorker worker = + new TestKubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + + worker.createVpcTierAclRules(network); + + Assert.assertEquals(0, worker.provisionAclRuleCalls); + } + + @Test + public void provisionFirewallRulesReusesOnlyManifestedRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Account account = Mockito.mock(Account.class); + FirewallRuleVO ownedRule = Mockito.mock(FirewallRuleVO.class); + KubernetesClusterFirewallRuleMapVO ownership = mockFirewallOwnership(20L, + KubernetesClusterActionWorker.API_FIREWALL_ROLE, KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(account.getDomainId()).thenReturn(40L); + Mockito.when(kubernetesClusterMock.getNetworkId()).thenReturn(20L); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.API_FIREWALL_ROLE)).thenReturn(ownership); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByFirewallRuleId(20L)).thenReturn(ownership); + Mockito.when(firewallRulesDaoMock.findById(20L)).thenReturn(ownedRule); + Mockito.when(ownedRule.getId()).thenReturn(20L); + Mockito.when(ownedRule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(ownedRule.getSourceIpAddressId()).thenReturn(10L); + Mockito.when(ownedRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(ownedRule.getSourcePortEnd()).thenReturn(6443); + Mockito.when(ownedRule.getNetworkId()).thenReturn(20L); + Mockito.when(ownedRule.getAccountId()).thenReturn(30L); + Mockito.when(ownedRule.getDomainId()).thenReturn(40L); + Mockito.when(ownedRule.getPurpose()).thenReturn(FirewallRule.Purpose.Firewall); + Mockito.when(ownedRule.getTrafficType()).thenReturn(FirewallRule.TrafficType.Ingress); + Mockito.when(ownedRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(ownedRule.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS)); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.update(Mockito.anyLong(), Mockito.any())).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.provisionFirewallRules(publicIp, account, 6443, 6443, + KubernetesClusterActionWorker.API_FIREWALL_ROLE); + + Mockito.verify(firewallServiceMock).applyIngressFwRules(10L, account); + Mockito.verify(firewallServiceMock, Mockito.never()).createIngressFirewallRule(Mockito.any()); + } + + @Test(expected = CloudRuntimeException.class) + public void ownershipForAnotherClusterFailsClosed() { + KubernetesClusterFirewallRuleMapVO foreignOwnership = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Mockito.when(foreignOwnership.getClusterId()).thenReturn(2L); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByFirewallRuleId(20L)).thenReturn(foreignOwnership); + + kubernetesClusterResourceModifierActionWorker.recordManagedFirewallRule(20L, + KubernetesClusterActionWorker.API_FIREWALL_ROLE); + } + + @Test(expected = CloudRuntimeException.class) + public void firewallRuleCreationFailsWhenOwnershipCannotBePersisted() { + kubernetesClusterResourceModifierActionWorker.recordManagedFirewallRule(20L, + KubernetesClusterActionWorker.API_FIREWALL_ROLE); + } + + @Test(expected = CloudRuntimeException.class) + public void networkAclItemCreationFailsWhenOwnershipCannotBePersisted() { + kubernetesClusterResourceModifierActionWorker.recordManagedNetworkAclItem(20L, + KubernetesClusterActionWorker.API_ACL_ROLE); + } + + @Test + public void deleteManagedNetworkRulesNotInDeletesOnlyStaleOwnership() throws Exception { + Network network = Mockito.mock(Network.class); + String desiredRole = KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.toLogicalRole(40L); + String staleRole = KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.toLogicalRole(41L); + KubernetesClusterFirewallRuleMapVO desired = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + KubernetesClusterFirewallRuleMapVO stale = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Mockito.when(desired.getFirewallRuleId()).thenReturn(20L); + Mockito.when(desired.getLogicalRole()).thenReturn(desiredRole); + Mockito.when(stale.getFirewallRuleId()).thenReturn(21L); + Mockito.when(stale.getLogicalRole()).thenReturn(staleRole); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.listByClusterId(1L)).thenReturn(List.of(desired, stale)); + KubernetesClusterResourceModifierActionWorker spyWorker = Mockito.spy(kubernetesClusterResourceModifierActionWorker); + Mockito.doNothing().when(spyWorker).deleteManagedFirewallRuleByRole(Mockito.any(), Mockito.eq(network)); + + spyWorker.deleteManagedNetworkRulesNotIn(Set.of(desiredRole), network); + + Mockito.verify(spyWorker).deleteManagedFirewallRuleByRole(stale, network); + Mockito.verify(spyWorker, Mockito.never()).deleteManagedFirewallRuleByRole(desired, network); + } + + @Test + public void provisionPortForwardingRuleReusesOnlyManifestedRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Network network = Mockito.mock(Network.class); + Account account = Mockito.mock(Account.class); + Nic nic = Mockito.mock(Nic.class); + PortForwardingRuleVO ownedRule = Mockito.mock(PortForwardingRuleVO.class); + KubernetesClusterFirewallRuleMapVO ownership = mockFirewallOwnership(21L, + KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 40L, + KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + kubernetesClusterResourceModifierActionWorker.networkModel = Mockito.mock(NetworkModel.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(account.getDomainId()).thenReturn(40L); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 40L)).thenReturn(ownership); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByFirewallRuleId(21L)).thenReturn(ownership); + Mockito.when(portForwardingRulesDaoMock.findById(21L)).thenReturn(ownedRule); + Mockito.when(ownedRule.getId()).thenReturn(21L); + Mockito.when(ownedRule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(ownedRule.getSourceIpAddressId()).thenReturn(10L); + Mockito.when(ownedRule.getSourcePortStart()).thenReturn(2222); + Mockito.when(ownedRule.getSourcePortEnd()).thenReturn(2222); + Mockito.when(ownedRule.getDestinationPortStart()).thenReturn(22); + Mockito.when(ownedRule.getDestinationPortEnd()).thenReturn(22); + Mockito.when(ownedRule.getVirtualMachineId()).thenReturn(40L); + Mockito.when(ownedRule.getNetworkId()).thenReturn(20L); + Mockito.when(ownedRule.getAccountId()).thenReturn(30L); + Mockito.when(ownedRule.getDomainId()).thenReturn(40L); + Mockito.when(ownedRule.getPurpose()).thenReturn(FirewallRule.Purpose.PortForwarding); + Mockito.when(ownedRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(ownedRule.getDestinationIpAddress()).thenReturn(new Ip("10.1.1.10")); + Mockito.when(kubernetesClusterResourceModifierActionWorker.networkModel.getNicInNetwork(40L, 20L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("10.1.1.10"); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.update(Mockito.anyLong(), Mockito.any())).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.provisionPublicIpPortForwardingRule(publicIp, network, account, 40L, 2222, 22); + + Mockito.verify(rulesServiceMock).applyPortForwardingRules(10L, account); + Mockito.verify(portForwardingRulesDaoMock, Mockito.never()).persist(Mockito.any()); + } + + @Test + public void provisionPortForwardingRuleRecordsNewRuleOwnership() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Network network = Mockito.mock(Network.class); + Account account = Mockito.mock(Account.class); + Nic nic = Mockito.mock(Nic.class); + PortForwardingRuleVO createdRule = Mockito.mock(PortForwardingRuleVO.class); + KubernetesClusterFirewallRuleMapVO ownership = mockFirewallOwnership(41L, + KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 40L, + KubernetesClusterNetworkRuleLifecycleState.PENDING_APPLY); + kubernetesClusterResourceModifierActionWorker.networkModel = Mockito.mock(NetworkModel.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(account.getDomainId()).thenReturn(40L); + Mockito.when(kubernetesClusterResourceModifierActionWorker.networkModel.getNicInNetwork(40L, 20L)).thenReturn(nic); + Mockito.when(nic.getIPv4Address()).thenReturn("10.1.1.10"); + Mockito.when(portForwardingRulesDaoMock.listByIpAndNotRevoked(10L)).thenReturn(List.of()); + Mockito.when(portForwardingRulesDaoMock.persist(Mockito.any(PortForwardingRuleVO.class))).thenReturn(createdRule); + Mockito.when(createdRule.getId()).thenReturn(41L); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.persist(Mockito.any(KubernetesClusterFirewallRuleMapVO.class))).thenReturn(ownership); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByFirewallRuleId(41L)).thenReturn(null, ownership); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.update(Mockito.anyLong(), Mockito.any())).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.provisionPublicIpPortForwardingRule(publicIp, network, account, 40L, 2222, 22); + + Mockito.verify(kubernetesClusterFirewallRuleMapDaoMock).persist(Mockito.argThat(mapping -> mapping.getClusterId() == 1L + && mapping.getFirewallRuleId() == 41L + && (KubernetesClusterActionWorker.SSH_PORT_FORWARD_ROLE_PREFIX + 40L).equals(mapping.getLogicalRole()))); + Mockito.verify(rulesServiceMock).applyPortForwardingRules(10L, account); + } + + @Test + public void provisionVpcTierAclRuleReloadsTierBeforeReusingManifestedRule() throws Exception { + Network staleNetwork = Mockito.mock(Network.class); + com.cloud.network.dao.NetworkVO effectiveNetwork = Mockito.mock(com.cloud.network.dao.NetworkVO.class); + NetworkACLItemVO ownedRule = Mockito.mock(NetworkACLItemVO.class); + KubernetesClusterNetworkACLItemMapVO ownership = mockAclOwnership(22L, KubernetesClusterActionWorker.API_ACL_ROLE, + KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(staleNetwork.getId()).thenReturn(20L); + Mockito.when(effectiveNetwork.getNetworkACLId()).thenReturn(50L); + Mockito.when(networkDaoMock.findById(20L)).thenReturn(effectiveNetwork); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.API_ACL_ROLE)).thenReturn(ownership); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findByNetworkAclItemId(22L)).thenReturn(ownership); + Mockito.when(networkACLItemDaoMock.findById(22L)).thenReturn(ownedRule); + Mockito.when(ownedRule.getId()).thenReturn(22L); + Mockito.when(ownedRule.getState()).thenReturn(NetworkACLItem.State.Active); + Mockito.when(ownedRule.getAclId()).thenReturn(50L); + Mockito.when(ownedRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(ownedRule.getSourcePortEnd()).thenReturn(6443); + Mockito.when(ownedRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(ownedRule.getTrafficType()).thenReturn(NetworkACLItem.TrafficType.Ingress); + Mockito.when(ownedRule.getAction()).thenReturn(NetworkACLItem.Action.Allow); + Mockito.when(ownedRule.getSourceCidrList()).thenReturn(List.of(NetUtils.ALL_IP4_CIDRS, NetUtils.ALL_IP6_CIDRS)); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.update(Mockito.anyLong(), Mockito.any())).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.provisionVpcTierAllowPortACLRule(staleNetwork, 6443, 6443, + KubernetesClusterActionWorker.API_ACL_ROLE); + + Mockito.verify(networkACLServiceMock).applyNetworkACL(50L); + Mockito.verify(networkACLServiceMock, Mockito.never()).createNetworkACLItem(Mockito.any()); + Mockito.verify(staleNetwork, Mockito.never()).getNetworkACLId(); + } + + @Test(expected = CloudRuntimeException.class) + public void deleteManagedAclItemRejectsRuleWhoseAclWasAttachedToAnotherTier() throws Exception { + Network network = Mockito.mock(Network.class); + com.cloud.network.dao.NetworkVO otherTier = Mockito.mock(com.cloud.network.dao.NetworkVO.class); + NetworkACLItemVO item = Mockito.mock(NetworkACLItemVO.class); + NetworkACLVO acl = Mockito.mock(NetworkACLVO.class); + KubernetesClusterNetworkACLItemMapVO ownership = mockAclOwnership(22L, + KubernetesClusterActionWorker.API_ACL_ROLE, KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(otherTier.getId()).thenReturn(21L); + Mockito.when(networkACLItemDaoMock.findById(22L)).thenReturn(item); + Mockito.when(item.getId()).thenReturn(22L); + Mockito.when(item.getAclId()).thenReturn(50L); + Mockito.when(networkACLDaoMock.findById(50L)).thenReturn(acl); + Mockito.when(acl.getVpcId()).thenReturn(10L); + Mockito.when(networkDaoMock.listByAclId(50L)).thenReturn(List.of(otherTier)); + + try { + kubernetesClusterResourceModifierActionWorker.deleteManagedNetworkAclItem(ownership, network); + } finally { + Mockito.verify(networkACLServiceMock, Mockito.never()).revokeNetworkACLItem(Mockito.anyLong()); + } + } + + @Test + public void deleteManagedAclItemForgetsOwnershipWhenItemIsAlreadyMissing() throws Exception { + Network network = Mockito.mock(Network.class); + KubernetesClusterNetworkACLItemMapVO ownership = mockAclOwnership(22L, + KubernetesClusterActionWorker.API_ACL_ROLE, KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findByNetworkAclItemId(22L)).thenReturn(ownership); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.update(1022L, ownership)).thenReturn(true); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findById(1022L)).thenReturn(ownership); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.remove(1022L)).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.deleteManagedNetworkAclItem(ownership, network); + + Mockito.verify(networkACLServiceMock, Mockito.never()).revokeNetworkACLItem(Mockito.anyLong()); + Mockito.verify(kubernetesClusterNetworkACLItemMapDaoMock).remove(1022L); + } + + @Test + public void deleteManagedAclItemRevokesExactItemAndForgetsOwnership() throws Exception { + com.cloud.network.dao.NetworkVO network = Mockito.mock(com.cloud.network.dao.NetworkVO.class); + NetworkACLItemVO item = Mockito.mock(NetworkACLItemVO.class); + NetworkACLVO acl = Mockito.mock(NetworkACLVO.class); + KubernetesClusterNetworkACLItemMapVO ownership = mockAclOwnership(22L, + KubernetesClusterActionWorker.API_ACL_ROLE, KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(network.getVpcId()).thenReturn(10L); + Mockito.when(item.getId()).thenReturn(22L); + Mockito.when(item.getAclId()).thenReturn(50L); + Mockito.when(networkACLItemDaoMock.findById(22L)).thenReturn(item, item, null); + Mockito.when(networkACLDaoMock.findById(50L)).thenReturn(acl); + Mockito.when(acl.getVpcId()).thenReturn(10L); + Mockito.when(networkDaoMock.listByAclId(50L)).thenReturn(List.of(network)); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findByNetworkAclItemId(22L)).thenReturn(ownership); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.update(1022L, ownership)).thenReturn(true); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.findById(1022L)).thenReturn(ownership); + Mockito.when(kubernetesClusterNetworkACLItemMapDaoMock.remove(1022L)).thenReturn(true); + Mockito.when(networkACLServiceMock.revokeNetworkACLItem(22L)).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.deleteManagedNetworkAclItem(ownership, network); + + Mockito.verify(networkACLServiceMock).revokeNetworkACLItem(22L); + Mockito.verify(kubernetesClusterNetworkACLItemMapDaoMock).remove(1022L); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionLoadBalancerRuleRejectsAnOverlappingRuleWithDifferentShape() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + LoadBalancerVO existingRule = Mockito.mock(LoadBalancerVO.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(loadBalancerDaoMock.listByIpAddress(10L)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getId()).thenReturn(60L); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(6443); + kubernetesClusterResourceModifierActionWorker.provisionLoadBalancerRule(publicIp, Mockito.mock(Network.class), Mockito.mock(Account.class), + List.of(), 6443); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionLoadBalancerRuleRejectsUnownedLegacyRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Network network = Mockito.mock(Network.class); + Account account = Mockito.mock(Account.class); + LoadBalancerVO existingRule = Mockito.mock(LoadBalancerVO.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(network.getId()).thenReturn(20L); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(loadBalancerDaoMock.listByIpAddress(10L)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(6443); + Mockito.when(existingRule.getDefaultPortStart()).thenReturn(6443); + Mockito.when(existingRule.getDefaultPortEnd()).thenReturn(6443); + Mockito.when(existingRule.getNetworkId()).thenReturn(20L); + Mockito.when(existingRule.getAccountId()).thenReturn(30L); + Mockito.when(existingRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(existingRule.getLbProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(existingRule.getName()).thenReturn("api-lb"); + Mockito.when(existingRule.getAlgorithm()).thenReturn("roundrobin"); + kubernetesClusterResourceModifierActionWorker.provisionLoadBalancerRule(publicIp, network, account, List.of(), 6443); + Mockito.verify(loadBalancingRulesServiceMock, Mockito.never()).createPublicLoadBalancerRule(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyLong(), Mockito.any(), Mockito.any(), + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.any(), Mockito.anyBoolean()); + } + + @Test + public void provisionLoadBalancerRuleReusesOnlyManifestedRule() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + Network network = Mockito.mock(Network.class); + LoadBalancerVO ownedRule = Mockito.mock(LoadBalancerVO.class); + KubernetesClusterFirewallRuleMapVO ownership = mockFirewallOwnership(23L, + KubernetesClusterActionWorker.API_LOAD_BALANCER_ROLE, KubernetesClusterNetworkRuleLifecycleState.ACTIVE); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByClusterIdAndLogicalRole(1L, + KubernetesClusterActionWorker.API_LOAD_BALANCER_ROLE)).thenReturn(ownership); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.findByFirewallRuleId(23L)).thenReturn(ownership); + Mockito.when(loadBalancerDaoMock.findById(23L)).thenReturn(ownedRule); + Mockito.when(ownedRule.getState()).thenReturn(FirewallRule.State.Active); + Mockito.when(ownedRule.getSourceIpAddressId()).thenReturn(10L); + Mockito.when(ownedRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(ownedRule.getSourcePortEnd()).thenReturn(6443); + Mockito.when(ownedRule.getDefaultPortStart()).thenReturn(6443); + Mockito.when(ownedRule.getDefaultPortEnd()).thenReturn(6443); + Mockito.when(ownedRule.getNetworkId()).thenReturn(20L); + Mockito.when(ownedRule.getAccountId()).thenReturn(30L); + Mockito.when(ownedRule.getProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(ownedRule.getLbProtocol()).thenReturn(NetUtils.TCP_PROTO); + Mockito.when(ownedRule.getName()).thenReturn("api-lb"); + Mockito.when(ownedRule.getAlgorithm()).thenReturn("roundrobin"); + Mockito.when(ownedRule.getId()).thenReturn(23L); + Mockito.when(network.getId()).thenReturn(20L); + Account account = Mockito.mock(Account.class); + Mockito.when(account.getId()).thenReturn(30L); + Mockito.when(loadBalancerVMMapDaoMock.listByLoadBalancerId(23L, false)).thenReturn(List.of()); + Mockito.when(loadBalancingRulesServiceMock.applyLoadBalancerConfig(23L)).thenReturn(true); + Mockito.when(kubernetesClusterFirewallRuleMapDaoMock.update(Mockito.anyLong(), Mockito.any())).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.provisionLoadBalancerRule(publicIp, network, account, List.of(), 6443); + + Mockito.verify(loadBalancingRulesServiceMock).applyLoadBalancerConfig(23L); + Mockito.verify(loadBalancingRulesServiceMock, Mockito.never()).createPublicLoadBalancerRule(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyLong(), Mockito.any(), Mockito.any(), + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.any(), Mockito.anyBoolean()); + } + + @Test(expected = NetworkRuleConflictException.class) + public void provisionLoadBalancerRuleRejectsAnOverlappingRuleWithMinimalMetadata() throws Exception { + IpAddress publicIp = Mockito.mock(IpAddress.class); + LoadBalancerVO existingRule = Mockito.mock(LoadBalancerVO.class); + Mockito.when(publicIp.getId()).thenReturn(10L); + Mockito.when(loadBalancerDaoMock.listByIpAddress(10L)).thenReturn(List.of(existingRule)); + Mockito.when(existingRule.getSourcePortStart()).thenReturn(6443); + Mockito.when(existingRule.getSourcePortEnd()).thenReturn(6443); + + kubernetesClusterResourceModifierActionWorker.provisionLoadBalancerRule(publicIp, Mockito.mock(Network.class), Mockito.mock(Account.class), List.of(), 6443); + } + + private KubernetesClusterFirewallRuleMapVO mockFirewallOwnership(long ruleId, String logicalRole, + KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + KubernetesClusterFirewallRuleMapVO ownership = Mockito.mock(KubernetesClusterFirewallRuleMapVO.class); + Mockito.when(ownership.getId()).thenReturn(ruleId + 1000L); + Mockito.when(ownership.getClusterId()).thenReturn(1L); + Mockito.when(ownership.getFirewallRuleId()).thenReturn(ruleId); + Mockito.when(ownership.getLifecycleState()).thenReturn(lifecycleState); + return ownership; + } + + private KubernetesClusterNetworkACLItemMapVO mockAclOwnership(long itemId, String logicalRole, + KubernetesClusterNetworkRuleLifecycleState lifecycleState) { + KubernetesClusterNetworkACLItemMapVO ownership = Mockito.mock(KubernetesClusterNetworkACLItemMapVO.class); + Mockito.when(ownership.getId()).thenReturn(itemId + 1000L); + Mockito.when(ownership.getClusterId()).thenReturn(1L); + Mockito.when(ownership.getNetworkAclItemId()).thenReturn(itemId); + Mockito.when(ownership.getLogicalRole()).thenReturn(logicalRole); + Mockito.when(ownership.getLifecycleState()).thenReturn(lifecycleState); + return ownership; + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..01ed93557393 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -17,9 +17,14 @@ package com.cloud.kubernetes.cluster.actionworkers; import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleRole; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.network.Network; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -37,6 +42,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Set; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; @@ -53,6 +59,8 @@ public class KubernetesClusterScaleWorkerTest { @Mock private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock private UserVmDao userVmDao; private KubernetesClusterScaleWorker worker; @@ -61,6 +69,7 @@ public class KubernetesClusterScaleWorkerTest { @Before public void setUp() { + clusterManager.kubernetesClusterDao = kubernetesClusterDao; worker = new KubernetesClusterScaleWorker(kubernetesCluster, clusterManager); worker.serviceOfferingDao = serviceOfferingDao; worker.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; @@ -187,4 +196,60 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + @Test + public void testScaleCleanupUsesCurrentOwnershipRoles() throws Exception { + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Network network = Mockito.mock(Network.class); + KubernetesClusterVmMapVO controlNode = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(controlNode.isControlNode()).thenReturn(true); + Mockito.when(controlNode.getVmId()).thenReturn(11L); + Mockito.when(clusterManager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(null); + Mockito.doReturn(List.of(controlNode)).when(spyWorker).getKubernetesClusterVMMaps(); + Mockito.doNothing().when(spyWorker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + + spyWorker.deleteStaleManagedNetworkRulesForScale(network); + + Set expectedRoles = Set.of( + KubernetesClusterNetworkRuleRole.API_FIREWALL.name(), + KubernetesClusterNetworkRuleRole.SSH_FIREWALL.name(), + KubernetesClusterNetworkRuleRole.API_LOAD_BALANCER.name(), + KubernetesClusterNetworkRuleRole.SSH_PORT_FORWARD.toLogicalRole(11L)); + Mockito.verify(spyWorker).deleteManagedNetworkRulesNotIn(expectedRoles, network); + } + + @Test + public void testScaleCleanupRemovesManagedMappingsFromDirectNetwork() throws Exception { + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Network network = Mockito.mock(Network.class); + KubernetesClusterVO persistedCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + Mockito.when(clusterManager.isDirectAccess(network)).thenReturn(true); + Mockito.when(kubernetesClusterDao.findById(10L)).thenReturn(persistedCluster); + Mockito.when(persistedCluster.getNetworkRuleOwnershipState()) + .thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + Mockito.doReturn(List.of()).when(spyWorker).getKubernetesClusterVMMaps(); + Mockito.doNothing().when(spyWorker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + + spyWorker.deleteStaleManagedNetworkRulesForScale(network); + + Mockito.verify(spyWorker).deleteManagedNetworkRulesNotIn(Set.of(), network); + } + + @Test + public void testScaleCleanupLeavesLegacyDirectNetworkUnownedRulesUntouched() throws Exception { + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Network network = Mockito.mock(Network.class); + KubernetesClusterVO persistedCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + Mockito.when(clusterManager.isDirectAccess(network)).thenReturn(true); + Mockito.when(kubernetesClusterDao.findById(10L)).thenReturn(persistedCluster); + Mockito.when(persistedCluster.getNetworkRuleOwnershipState()) + .thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + + spyWorker.deleteStaleManagedNetworkRulesForScale(network); + + Mockito.verify(spyWorker, Mockito.never()).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.any()); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java new file mode 100644 index 000000000000..fd7476e74ed8 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorkerTest.java @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.List; +import java.util.Set; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNetworkRuleOwnershipState; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.uservm.UserVm; +import com.cloud.vm.UserVmVO; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.vm.dao.UserVmDao; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterStartWorkerTest { + + @Mock + private KubernetesClusterManagerImpl manager; + @Mock + private KubernetesCluster cluster; + @Mock + private KubernetesClusterDao clusterDao; + @Mock + private KubernetesClusterDetailsDao clusterDetailsDao; + @Mock + private KubernetesClusterVmMapDao clusterVmMapDao; + @Mock + private KubernetesSupportedVersionDao supportedVersionDao; + @Mock + private AccountDao accountDao; + @Mock + private NetworkDao networkDao; + @Mock + private UserVmDao userVmDao; + @Mock + private NetworkOfferingDao networkOfferingDao; + + private KubernetesClusterStartWorker worker; + + @Before + public void setUp() { + manager.kubernetesClusterDao = clusterDao; + manager.kubernetesClusterDetailsDao = clusterDetailsDao; + manager.kubernetesClusterVmMapDao = clusterVmMapDao; + manager.kubernetesSupportedVersionDao = supportedVersionDao; + worker = Mockito.spy(new KubernetesClusterStartWorker(cluster, manager)); + worker.accountDao = accountDao; + worker.networkDao = networkDao; + worker.userVmDao = userVmDao; + worker.networkOfferingDao = networkOfferingDao; + Mockito.when(cluster.getAccountId()).thenReturn(2L); + Mockito.when(cluster.getNetworkId()).thenReturn(3L); + } + + @Test + public void reconcileDirectAccessNetworkIsANoOp() throws Exception { + AccountVO owner = Mockito.mock(AccountVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + Mockito.when(accountDao.findById(2L)).thenReturn(owner); + Mockito.when(networkDao.findById(3L)).thenReturn(network); + Mockito.when(manager.isDirectAccess(network)).thenReturn(true); + Mockito.doNothing().when(worker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + + Assert.assertTrue(worker.reconcileKubernetesClusterNetworkRules()); + + Mockito.verify(worker, Mockito.never()).getKubernetesClusterVMMaps(); + Mockito.verify(worker).deleteManagedNetworkRulesNotIn(Mockito.eq(java.util.Collections.emptySet()), Mockito.eq(network)); + } + + @Test + public void reconcileVpcTierDelegatesToVpcRuleProvisioning() throws Exception { + AccountVO owner = Mockito.mock(AccountVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + IpAddress publicIp = Mockito.mock(IpAddress.class); + UserVmVO controlVm = Mockito.mock(UserVmVO.class); + KubernetesClusterVmMapVO controlMap = new KubernetesClusterVmMapVO(1L, 12L, true); + Mockito.when(accountDao.findById(2L)).thenReturn(owner); + Mockito.when(networkDao.findById(3L)).thenReturn(network); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(4L); + Mockito.when(network.getNetworkACLId()).thenReturn(com.cloud.network.vpc.NetworkACL.DEFAULT_ALLOW); + Mockito.when(network.getNetworkOfferingId()).thenReturn(5L); + NetworkOfferingVO offering = Mockito.mock(NetworkOfferingVO.class); + Mockito.when(networkOfferingDao.findById(5L)).thenReturn(offering); + Mockito.doReturn(List.of(controlMap)).when(worker).getKubernetesClusterVMMaps(); + Mockito.when(userVmDao.findById(12L)).thenReturn(controlVm); + Mockito.when(controlVm.getId()).thenReturn(12L); + Mockito.doReturn(publicIp).when(worker).getPublicIp(network); + Mockito.when(publicIp.getAddress()).thenReturn(new Ip("203.0.113.10")); + Mockito.doNothing().when(worker).setupKubernetesClusterVpcTierRules(publicIp, network, List.of(12L)); + Mockito.doNothing().when(worker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + + Assert.assertTrue(worker.reconcileKubernetesClusterNetworkRules()); + + org.mockito.InOrder ordering = Mockito.inOrder(worker); + ordering.verify(worker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + ordering.verify(worker).setupKubernetesClusterVpcTierRules(publicIp, network, List.of(12L)); + } + + @SuppressWarnings("unchecked") + @Test + public void reconcileOrdersControlNodesBeforeWorkersAndHandlesEtcdSeparately() throws Exception { + AccountVO owner = Mockito.mock(AccountVO.class); + NetworkVO network = Mockito.mock(NetworkVO.class); + IpAddress publicIp = Mockito.mock(IpAddress.class); + UserVmVO workerVm = Mockito.mock(UserVmVO.class); + UserVmVO etcdVm = Mockito.mock(UserVmVO.class); + UserVmVO controlVm = Mockito.mock(UserVmVO.class); + KubernetesClusterVmMapVO workerMap = new KubernetesClusterVmMapVO(1L, 10L, false); + KubernetesClusterVmMapVO etcdMap = new KubernetesClusterVmMapVO(1L, 11L, false); + etcdMap.setEtcdNode(true); + KubernetesClusterVmMapVO controlMap = new KubernetesClusterVmMapVO(1L, 12L, true); + Mockito.when(accountDao.findById(2L)).thenReturn(owner); + Mockito.when(networkDao.findById(3L)).thenReturn(network); + Mockito.when(manager.isDirectAccess(network)).thenReturn(false); + Mockito.when(network.getVpcId()).thenReturn(null); + Mockito.doReturn(List.of(workerMap, etcdMap, controlMap)).when(worker).getKubernetesClusterVMMaps(); + Mockito.when(userVmDao.findById(10L)).thenReturn(workerVm); + Mockito.when(userVmDao.findById(11L)).thenReturn(etcdVm); + Mockito.when(userVmDao.findById(12L)).thenReturn(controlVm); + Mockito.doReturn(publicIp).when(worker).getPublicIp(network); + Mockito.when(publicIp.getAddress()).thenReturn(new Ip("203.0.113.10")); + Mockito.doNothing().when(worker).setupKubernetesClusterNetworkRules(Mockito.eq(network), Mockito.anyList(), Mockito.eq(publicIp)); + Mockito.doNothing().when(worker).setupKubernetesEtcdNetworkRules(Mockito.anyList(), Mockito.eq(network), Mockito.eq(publicIp)); + Mockito.doNothing().when(worker).deleteManagedNetworkRulesNotIn(Mockito.anySet(), Mockito.eq(network)); + + Assert.assertTrue(worker.reconcileKubernetesClusterNetworkRules()); + + ArgumentCaptor> clusterVms = ArgumentCaptor.forClass(List.class); + ArgumentCaptor> etcdVms = ArgumentCaptor.forClass(List.class); + Mockito.verify(worker).setupKubernetesClusterNetworkRules(Mockito.eq(network), clusterVms.capture(), Mockito.eq(publicIp)); + Mockito.verify(worker).setupKubernetesEtcdNetworkRules(etcdVms.capture(), Mockito.eq(network), Mockito.eq(publicIp)); + Assert.assertEquals(List.of(controlVm, workerVm), clusterVms.getValue()); + Assert.assertEquals(List.of(etcdVm), etcdVms.getValue()); + ArgumentCaptor> desiredRoles = ArgumentCaptor.forClass(Set.class); + Mockito.verify(worker).deleteManagedNetworkRulesNotIn(desiredRoles.capture(), Mockito.eq(network)); + Assert.assertEquals(Set.of("API_FIREWALL", "SSH_FIREWALL", "ETCD_SSH_FIREWALL:11", "API_LOAD_BALANCER", + "SSH_PORT_FORWARD:10", "SSH_PORT_FORWARD:11", "SSH_PORT_FORWARD:12"), desiredRoles.getValue()); + } + + @Test + public void managedClusterReconcilesRulesBeforeReadiness() { + KubernetesClusterVO currentCluster = mockManagedCluster(); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + Mockito.doReturn(true).when(worker).reconcileKubernetesClusterNetworkRules(); + + Assert.assertTrue(worker.reconcileManagedNetworkRulesBeforeReadiness()); + + Mockito.verify(worker).reconcileKubernetesClusterNetworkRules(); + } + + @Test + public void legacyClusterDoesNotClaimRulesDuringReadiness() { + KubernetesClusterVO currentCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(currentCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(currentCluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.LEGACY_UNMANAGED); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + + Assert.assertTrue(worker.reconcileManagedNetworkRulesBeforeReadiness()); + + Mockito.verify(worker, Mockito.never()).reconcileKubernetesClusterNetworkRules(); + } + + @Test + public void stoppedClusterStartsVmsThenReconcilesBeforeEndpointReadiness() { + KubernetesClusterVO currentCluster = mockManagedCluster(); + Mockito.when(cluster.getName()).thenReturn("cluster"); + Mockito.when(cluster.getEndpoint()).thenReturn("not-a-url"); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + Mockito.doNothing().when(worker).init(); + Mockito.doReturn(true).when(worker).stateTransitTo(Mockito.anyLong(), Mockito.any()); + Mockito.doNothing().when(worker).startKubernetesClusterVMs(4L, 5L); + Mockito.doReturn(true).when(worker).reconcileKubernetesClusterNetworkRules(); + Mockito.doThrow(new CloudRuntimeException("expected endpoint failure")).when(worker).logTransitStateAndThrow( + Mockito.any(), Mockito.anyString(), Mockito.anyLong(), Mockito.any()); + + try { + worker.startStoppedKubernetesCluster(4L, 5L); + Assert.fail("Expected endpoint validation to fail"); + } catch (CloudRuntimeException expected) { + Assert.assertEquals("expected endpoint failure", expected.getMessage()); + } + + org.mockito.InOrder ordering = Mockito.inOrder(worker); + ordering.verify(worker).startKubernetesClusterVMs(4L, 5L); + ordering.verify(worker).reconcileKubernetesClusterNetworkRules(); + } + + @Test + public void stoppedClusterDoesNotProbeReadinessWhenManagedRuleReconciliationFails() { + KubernetesClusterVO currentCluster = mockManagedCluster(); + Mockito.when(cluster.getName()).thenReturn("cluster"); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + Mockito.doNothing().when(worker).init(); + Mockito.doReturn(true).when(worker).stateTransitTo(Mockito.anyLong(), Mockito.any()); + Mockito.doNothing().when(worker).startKubernetesClusterVMs(4L, 5L); + Mockito.doReturn(false).when(worker).reconcileKubernetesClusterNetworkRules(); + Mockito.doThrow(new CloudRuntimeException("expected reconciliation failure")).when(worker).logTransitStateAndThrow( + Mockito.any(), Mockito.anyString(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + try { + worker.startStoppedKubernetesCluster(4L, 5L); + Assert.fail("Expected reconciliation failure"); + } catch (CloudRuntimeException expected) { + Assert.assertEquals("expected reconciliation failure", expected.getMessage()); + } + + Mockito.verify(worker, Mockito.never()).getKubernetesClusterServerIpSshPort(Mockito.any()); + } + + @Test + public void alertRecoveryReconcilesManagedRulesBeforeInspectingNodes() { + KubernetesClusterVO currentCluster = mockManagedCluster(); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + Mockito.doNothing().when(worker).init(); + Mockito.doReturn(true).when(worker).reconcileKubernetesClusterNetworkRules(); + Mockito.doReturn(java.util.Collections.emptyList()).when(worker).getKubernetesClusterVMMaps(); + + Assert.assertFalse(worker.reconcileAlertCluster()); + + org.mockito.InOrder ordering = Mockito.inOrder(worker); + ordering.verify(worker).reconcileKubernetesClusterNetworkRules(); + ordering.verify(worker).getKubernetesClusterVMMaps(); + } + + @Test + public void alertRecoveryStopsBeforeNodeChecksWhenManagedRuleReconciliationFails() { + KubernetesClusterVO currentCluster = mockManagedCluster(); + Mockito.when(clusterDao.findById(1L)).thenReturn(currentCluster); + Mockito.doNothing().when(worker).init(); + Mockito.doReturn(false).when(worker).reconcileKubernetesClusterNetworkRules(); + + Assert.assertFalse(worker.reconcileAlertCluster()); + + Mockito.verify(worker, Mockito.never()).getKubernetesClusterVMMaps(); + } + + private KubernetesClusterVO mockManagedCluster() { + KubernetesClusterVO currentCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(currentCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(currentCluster.getNetworkRuleOwnershipState()).thenReturn(KubernetesClusterNetworkRuleOwnershipState.MANAGED); + return currentCluster; + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/KubernetesClusterNetworkRuleCmdTest.java b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/KubernetesClusterNetworkRuleCmdTest.java new file mode 100644 index 000000000000..56d6b24c8797 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/KubernetesClusterNetworkRuleCmdTest.java @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import java.lang.reflect.Field; +import java.util.List; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.cluster.AdoptKubernetesClusterNetworkRulesCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ReconcileKubernetesClusterNetworkRulesCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + +public class KubernetesClusterNetworkRuleCmdTest { + + private static final long CLUSTER_ID = 11L; + private static final long NETWORK_ID = 22L; + + @Test + public void testReconcileCommandSynchronizesOnClusterNetwork() { + ReconcileKubernetesClusterNetworkRulesCmd cmd = new ReconcileKubernetesClusterNetworkRulesCmd(); + cmd.kubernetesClusterService = mockClusterService(); + ReflectionTestUtils.setField(cmd, "id", CLUSTER_ID); + + Assert.assertEquals(BaseAsyncCmd.networkSyncObject, cmd.getSyncObjType()); + Assert.assertEquals(Long.valueOf(NETWORK_ID), cmd.getSyncObjId()); + } + + @Test + public void testAdoptCommandSynchronizesOnClusterNetwork() { + AdoptKubernetesClusterNetworkRulesCmd cmd = new AdoptKubernetesClusterNetworkRulesCmd(); + cmd.kubernetesClusterService = mockClusterService(); + ReflectionTestUtils.setField(cmd, "id", CLUSTER_ID); + + Assert.assertEquals(BaseAsyncCmd.networkSyncObject, cmd.getSyncObjType()); + Assert.assertEquals(Long.valueOf(NETWORK_ID), cmd.getSyncObjId()); + } + + @Test + public void testMissingClusterDoesNotCreateInvalidSyncQueue() { + ReconcileKubernetesClusterNetworkRulesCmd cmd = new ReconcileKubernetesClusterNetworkRulesCmd(); + cmd.kubernetesClusterService = Mockito.mock(KubernetesClusterService.class); + ReflectionTestUtils.setField(cmd, "id", CLUSTER_ID); + + Assert.assertNull(cmd.getSyncObjId()); + } + + @Test + public void testAllLifecycleCommandsSynchronizeWithNetworkRuleCommands() { + CreateKubernetesClusterCmd createCmd = new CreateKubernetesClusterCmd(); + createCmd.kubernetesClusterService = mockClusterService(); + createCmd.setEntityId(CLUSTER_ID); + Assert.assertEquals(BaseAsyncCmd.networkSyncObject, createCmd.getSyncObjType()); + Assert.assertEquals(Long.valueOf(NETWORK_ID), createCmd.getSyncObjId()); + + assertCommandSynchronizesOnClusterNetwork(new AddNodesToKubernetesClusterCmd(), "clusterId"); + assertCommandSynchronizesOnClusterNetwork(new RemoveNodesFromKubernetesClusterCmd(), "clusterId"); + assertCommandSynchronizesOnClusterNetwork(new DeleteKubernetesClusterCmd(), "id"); + assertCommandSynchronizesOnClusterNetwork(new ScaleKubernetesClusterCmd(), "id"); + assertCommandSynchronizesOnClusterNetwork(new StartKubernetesClusterCmd(), "id"); + assertCommandSynchronizesOnClusterNetwork(new StopKubernetesClusterCmd(), "id"); + assertCommandSynchronizesOnClusterNetwork(new UpgradeKubernetesClusterCmd(), "id"); + } + + @Test + public void testCreateCommandFallsBackToRequestedNetworkBeforeEntityLookup() { + CreateKubernetesClusterCmd cmd = new CreateKubernetesClusterCmd(); + cmd.kubernetesClusterService = Mockito.mock(KubernetesClusterService.class); + ReflectionTestUtils.setField(cmd, "networkId", NETWORK_ID); + + Assert.assertEquals(Long.valueOf(NETWORK_ID), cmd.getSyncObjId()); + } + + @Test + public void testLifecycleClusterIdsRequireOperateAccessBeforeQueueing() throws Exception { + List> commandClasses = List.of(DeleteKubernetesClusterCmd.class, + ScaleKubernetesClusterCmd.class, StartKubernetesClusterCmd.class, + StopKubernetesClusterCmd.class, UpgradeKubernetesClusterCmd.class, + ReconcileKubernetesClusterNetworkRulesCmd.class); + + for (Class commandClass : commandClasses) { + Field idField = commandClass.getDeclaredField("id"); + ACL acl = idField.getAnnotation(ACL.class); + Assert.assertNotNull(commandClass.getSimpleName() + " must authorize its cluster before queueing", acl); + Assert.assertEquals(SecurityChecker.AccessType.OperateEntry, acl.accessType()); + } + } + + @Test + public void testMissingClusterDoesNotCreateLifecycleSyncQueue() { + AddNodesToKubernetesClusterCmd cmd = new AddNodesToKubernetesClusterCmd(); + cmd.kubernetesClusterService = Mockito.mock(KubernetesClusterService.class); + ReflectionTestUtils.setField(cmd, "clusterId", CLUSTER_ID); + + Assert.assertNull(cmd.getSyncObjId()); + } + + private void assertCommandSynchronizesOnClusterNetwork(BaseAsyncCmd cmd, String clusterIdField) { + ReflectionTestUtils.setField(cmd, "kubernetesClusterService", mockClusterService()); + ReflectionTestUtils.setField(cmd, clusterIdField, CLUSTER_ID); + + Assert.assertEquals(BaseAsyncCmd.networkSyncObject, cmd.getSyncObjType()); + Assert.assertEquals(Long.valueOf(NETWORK_ID), cmd.getSyncObjId()); + } + + private KubernetesClusterService mockClusterService() { + KubernetesClusterService service = Mockito.mock(KubernetesClusterService.class); + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getNetworkId()).thenReturn(NETWORK_ID); + Mockito.when(service.findById(CLUSTER_ID)).thenReturn(cluster); + return service; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxAnswer.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxAnswer.java index a667adda7945..f2a629fafd84 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxAnswer.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxAnswer.java @@ -22,6 +22,7 @@ public class NsxAnswer extends Answer { private boolean objectExists; + private boolean endpointMayBeInUse; public NsxAnswer(final Command command, final boolean success, final String details) { super(command, success, details); @@ -38,4 +39,12 @@ public boolean isObjectExistent() { public void setObjectExists(boolean objectExisted) { this.objectExists = objectExisted; } + + public boolean isEndpointMayBeInUse() { + return endpointMayBeInUse; + } + + public void setEndpointMayBeInUse(boolean endpointMayBeInUse) { + this.endpointMayBeInUse = endpointMayBeInUse; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxVrfGatewayValidationAnswer.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxVrfGatewayValidationAnswer.java new file mode 100644 index 000000000000..2d8b85b5231d --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/NsxVrfGatewayValidationAnswer.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack; + +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; + +public class NsxVrfGatewayValidationAnswer extends NsxAnswer { + + private final String edgeClusterPath; + private final String parentTier0Path; + + public NsxVrfGatewayValidationAnswer(ValidateNsxVrfGatewayCommand command, + boolean success, String details, String edgeClusterPath, String parentTier0Path) { + super(command, success, details); + this.edgeClusterPath = edgeClusterPath; + this.parentTier0Path = parentTier0Path; + } + + public NsxVrfGatewayValidationAnswer(ValidateNsxVrfGatewayCommand command, Exception exception) { + super(command, exception); + this.edgeClusterPath = null; + this.parentTier0Path = null; + } + + public String getEdgeClusterPath() { + return edgeClusterPath; + } + + public String getParentTier0Path() { + return parentTier0Path; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxSegmentCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxSegmentCommand.java index b4b86bd640a6..e03fb1751190 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxSegmentCommand.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxSegmentCommand.java @@ -26,10 +26,21 @@ public class CreateNsxSegmentCommand extends NsxCommand { private String networkName; private String networkGateway; private String networkCidr; + private String ipDiscoveryProfileId; + private String macDiscoveryProfileId; + private String segmentSecurityProfileId; public CreateNsxSegmentCommand(long domainId, long accountId, long zoneId, Long vpcId, String vpcName, long networkId, String networkName, String networkGateway, String networkCidr) { + this(domainId, accountId, zoneId, vpcId, vpcName, networkId, networkName, networkGateway, + networkCidr, null, null, null); + } + + public CreateNsxSegmentCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, long networkId, String networkName, + String networkGateway, String networkCidr, String ipDiscoveryProfileId, + String macDiscoveryProfileId, String segmentSecurityProfileId) { super(domainId, accountId, zoneId); this.vpcId = vpcId; this.vpcName = vpcName; @@ -37,6 +48,9 @@ public CreateNsxSegmentCommand(long domainId, long accountId, long zoneId, this.networkName = networkName; this.networkGateway = networkGateway; this.networkCidr = networkCidr; + this.ipDiscoveryProfileId = ipDiscoveryProfileId; + this.macDiscoveryProfileId = macDiscoveryProfileId; + this.segmentSecurityProfileId = segmentSecurityProfileId; } public Long getVpcId() { @@ -63,6 +77,18 @@ public String getNetworkCidr() { return networkCidr; } + public String getIpDiscoveryProfileId() { + return ipDiscoveryProfileId; + } + + public String getMacDiscoveryProfileId() { + return macDiscoveryProfileId; + } + + public String getSegmentSecurityProfileId() { + return segmentSecurityProfileId; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxTier1GatewayCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxTier1GatewayCommand.java index 90e4b3a25bdd..aad7c3b3ad31 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxTier1GatewayCommand.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxTier1GatewayCommand.java @@ -24,6 +24,14 @@ public class CreateNsxTier1GatewayCommand extends NsxCommand { private String networkResourceName; private boolean isResourceVpc; private boolean sourceNatEnabled; + /** + * Tier-0 and edge cluster this gateway should be created under, resolved from the + * tenant's registered NSX VRF gateway. Both are null when the zone is not + * VRF-segregated, in which case the resource falls back to the zone-wide values it + * was configured with — which keeps every pre-VRF deployment byte-identical. + */ + private String tier0Gateway; + private String edgeCluster; public CreateNsxTier1GatewayCommand(long domainId, long accountId, long zoneId, Long networkResourceId, String networkResourceName, boolean isResourceVpc, @@ -51,6 +59,22 @@ public boolean isSourceNatEnabled() { return sourceNatEnabled; } + public String getTier0Gateway() { + return tier0Gateway; + } + + public void setTier0Gateway(String tier0Gateway) { + this.tier0Gateway = tier0Gateway; + } + + public String getEdgeCluster() { + return edgeCluster; + } + + public void setEdgeCluster(String edgeCluster) { + this.edgeCluster = edgeCluster; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnConnectionCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnConnectionCommand.java new file mode 100644 index 000000000000..050e475cd828 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnConnectionCommand.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.List; +import java.util.Objects; + +import com.cloud.agent.api.LogLevel; + +public class CreateNsxVpnConnectionCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + private String connectionUuid; + private String peerAddress; + @LogLevel(LogLevel.Log4jLevel.Off) + private String psk; + private String ikePolicy; + private String espPolicy; + private Long ikeLifetime; + private Long espLifetime; + private boolean dpdEnabled; + private String ikeVersion; + private boolean passive; + private List peerCidrs; + private String vtiLocalIp; + private String vtiPeerIp; + private int vtiPrefixLength; + private String vpcCidr; + private String localEndpointIp; + + public CreateNsxVpnConnectionCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, String connectionUuid, String peerAddress, + String psk, String ikePolicy, String espPolicy, Long ikeLifetime, + Long espLifetime, boolean dpdEnabled, String ikeVersion, boolean passive, + List peerCidrs, String vtiLocalIp, String vtiPeerIp, + int vtiPrefixLength, String vpcCidr, String localEndpointIp) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + this.connectionUuid = connectionUuid; + this.peerAddress = peerAddress; + this.psk = psk; + this.ikePolicy = ikePolicy; + this.espPolicy = espPolicy; + this.ikeLifetime = ikeLifetime; + this.espLifetime = espLifetime; + this.dpdEnabled = dpdEnabled; + this.ikeVersion = ikeVersion; + this.passive = passive; + this.peerCidrs = peerCidrs; + this.vtiLocalIp = vtiLocalIp; + this.vtiPeerIp = vtiPeerIp; + this.vtiPrefixLength = vtiPrefixLength; + this.vpcCidr = vpcCidr; + this.localEndpointIp = localEndpointIp; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + public String getConnectionUuid() { + return connectionUuid; + } + + public String getPeerAddress() { + return peerAddress; + } + + public String getPsk() { + return psk; + } + + public String getIkePolicy() { + return ikePolicy; + } + + public String getEspPolicy() { + return espPolicy; + } + + public Long getIkeLifetime() { + return ikeLifetime; + } + + public Long getEspLifetime() { + return espLifetime; + } + + public boolean isDpdEnabled() { + return dpdEnabled; + } + + public String getIkeVersion() { + return ikeVersion; + } + + public boolean isPassive() { + return passive; + } + + public List getPeerCidrs() { + return peerCidrs; + } + + public String getVtiLocalIp() { + return vtiLocalIp; + } + + public String getVtiPeerIp() { + return vtiPeerIp; + } + + public int getVtiPrefixLength() { + return vtiPrefixLength; + } + + public String getVpcCidr() { + return vpcCidr; + } + + public String getLocalEndpointIp() { + return localEndpointIp; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + CreateNsxVpnConnectionCommand that = (CreateNsxVpnConnectionCommand) o; + return dpdEnabled == that.dpdEnabled && passive == that.passive && vtiPrefixLength == that.vtiPrefixLength + && Objects.equals(vpcId, that.vpcId) && Objects.equals(vpcName, that.vpcName) + && Objects.equals(connectionUuid, that.connectionUuid) && Objects.equals(peerAddress, that.peerAddress) + && Objects.equals(psk, that.psk) && Objects.equals(ikePolicy, that.ikePolicy) + && Objects.equals(espPolicy, that.espPolicy) && Objects.equals(ikeLifetime, that.ikeLifetime) + && Objects.equals(espLifetime, that.espLifetime) && Objects.equals(ikeVersion, that.ikeVersion) + && Objects.equals(peerCidrs, that.peerCidrs) && Objects.equals(vtiLocalIp, that.vtiLocalIp) + && Objects.equals(vtiPeerIp, that.vtiPeerIp) && Objects.equals(vpcCidr, that.vpcCidr) + && Objects.equals(localEndpointIp, that.localEndpointIp); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName, connectionUuid, peerAddress, psk, ikePolicy, espPolicy, + ikeLifetime, espLifetime, dpdEnabled, ikeVersion, passive, peerCidrs, vtiLocalIp, vtiPeerIp, + vtiPrefixLength, vpcCidr, localEndpointIp); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnGatewayCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnGatewayCommand.java new file mode 100644 index 000000000000..9a381f5cb4b9 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/CreateNsxVpnGatewayCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class CreateNsxVpnGatewayCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + private String localEndpointIp; + + public CreateNsxVpnGatewayCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, String localEndpointIp) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + this.localEndpointIp = localEndpointIp; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + public String getLocalEndpointIp() { + return localEndpointIp; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + CreateNsxVpnGatewayCommand that = (CreateNsxVpnGatewayCommand) o; + return Objects.equals(vpcId, that.vpcId) && Objects.equals(vpcName, that.vpcName) && Objects.equals(localEndpointIp, that.localEndpointIp); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName, localEndpointIp); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnConnectionCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnConnectionCommand.java new file mode 100644 index 000000000000..83b0644e695f --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnConnectionCommand.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class DeleteNsxVpnConnectionCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + private String connectionUuid; + + public DeleteNsxVpnConnectionCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, String connectionUuid) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + this.connectionUuid = connectionUuid; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + public String getConnectionUuid() { + return connectionUuid; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + DeleteNsxVpnConnectionCommand that = (DeleteNsxVpnConnectionCommand) o; + return Objects.equals(vpcId, that.vpcId) && Objects.equals(vpcName, that.vpcName) + && Objects.equals(connectionUuid, that.connectionUuid); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName, connectionUuid); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnGatewayCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnGatewayCommand.java new file mode 100644 index 000000000000..cb5a97f9d344 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxVpnGatewayCommand.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class DeleteNsxVpnGatewayCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + + public DeleteNsxVpnGatewayCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + DeleteNsxVpnGatewayCommand that = (DeleteNsxVpnGatewayCommand) o; + return Objects.equals(vpcId, that.vpcId) && Objects.equals(vpcName, that.vpcName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/GetNsxVpnSessionStatusCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/GetNsxVpnSessionStatusCommand.java new file mode 100644 index 000000000000..0aed9e33f8f7 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/GetNsxVpnSessionStatusCommand.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class GetNsxVpnSessionStatusCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + private String connectionUuid; + + public GetNsxVpnSessionStatusCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, String connectionUuid) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + this.connectionUuid = connectionUuid; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + public String getConnectionUuid() { + return connectionUuid; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + GetNsxVpnSessionStatusCommand that = (GetNsxVpnSessionStatusCommand) o; + return Objects.equals(vpcId, that.vpcId) && Objects.equals(vpcName, that.vpcName) + && Objects.equals(connectionUuid, that.connectionUuid); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName, connectionUuid); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/UpdateNsxVpnConnectionStateCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/UpdateNsxVpnConnectionStateCommand.java new file mode 100644 index 000000000000..541065f86a04 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/UpdateNsxVpnConnectionStateCommand.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +import java.util.Objects; + +public class UpdateNsxVpnConnectionStateCommand extends NsxCommand { + + private Long vpcId; + private String vpcName; + private String connectionUuid; + private boolean enabled; + + public UpdateNsxVpnConnectionStateCommand(long domainId, long accountId, long zoneId, + Long vpcId, String vpcName, String connectionUuid, + boolean enabled) { + super(domainId, accountId, zoneId); + this.vpcId = vpcId; + this.vpcName = vpcName; + this.connectionUuid = connectionUuid; + this.enabled = enabled; + } + + public Long getVpcId() { + return vpcId; + } + + public String getVpcName() { + return vpcName; + } + + public String getConnectionUuid() { + return connectionUuid; + } + + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + UpdateNsxVpnConnectionStateCommand that = (UpdateNsxVpnConnectionStateCommand) o; + return enabled == that.enabled && Objects.equals(vpcId, that.vpcId) + && Objects.equals(vpcName, that.vpcName) + && Objects.equals(connectionUuid, that.connectionUuid); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), vpcId, vpcName, connectionUuid, enabled); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/ValidateNsxVrfGatewayCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/ValidateNsxVrfGatewayCommand.java new file mode 100644 index 000000000000..a3decafe499b --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/ValidateNsxVrfGatewayCommand.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.agent.api; + +public class ValidateNsxVrfGatewayCommand extends NsxCommand { + + private final String tier0Gateway; + private final String expectedParentTier0; + private final String expectedEdgeCluster; + + public ValidateNsxVrfGatewayCommand(long zoneId, String tier0Gateway, + String expectedParentTier0, String expectedEdgeCluster) { + super(0L, 0L, zoneId); + this.tier0Gateway = tier0Gateway; + this.expectedParentTier0 = expectedParentTier0; + this.expectedEdgeCluster = expectedEdgeCluster; + } + + public String getTier0Gateway() { + return tier0Gateway; + } + + public String getExpectedParentTier0() { + return expectedParentTier0; + } + + public String getExpectedEdgeCluster() { + return expectedEdgeCluster; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AddNsxVrfGatewayCmd.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AddNsxVrfGatewayCmd.java new file mode 100644 index 000000000000..be52ab130a8d --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AddNsxVrfGatewayCmd.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.api.response.VlanIpRangeResponse; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.service.NsxProviderService; + +/** + * Registers a VRF, or dedicated, tier-0 gateway that the operator has already staged in + * NSX. CloudStack never creates the gateway itself: its uplink interfaces and BGP peerings + * are provisioned alongside the physical network and cannot be automated from here. + */ +@APICommand(name = AddNsxVrfGatewayCmd.APINAME, description = "Registers an existing NSX VRF or dedicated tier-0 gateway with CloudStack, so tenant tier-1 gateways can be attached to it", + responseObject = NsxVrfGatewayResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, since = "4.23.0", authorized = {RoleType.Admin}) +public class AddNsxVrfGatewayCmd extends BaseCmd { + public static final String APINAME = "addNsxVrfGateway"; + + @Inject + NsxProviderService nsxProviderService; + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, + description = "the ID of the zone the tier-0 gateway belongs to") + private Long zoneId; + + @Parameter(name = ApiConstants.TIER0_GATEWAY, type = CommandType.STRING, required = true, + description = "name of the VRF or dedicated tier-0 gateway as it exists in NSX") + private String tier0Gateway; + + @Parameter(name = ApiConstants.EDGE_CLUSTER, type = CommandType.STRING, + description = "optional expected edge-cluster ID or path; CloudStack reads and stores the actual path from NSX") + private String edgeCluster; + + @Parameter(name = ApiConstants.PARENT_TIER0_GATEWAY, type = CommandType.STRING, + description = "parent tier-0 gateway when registering a VRF gateway; omit for a dedicated tier-0") + private String parentTier0Gateway; + + @Parameter(name = ApiConstants.VLAN_ID, type = CommandType.UUID, entityType = VlanIpRangeResponse.class, + required = true, description = "public IP range advertised by this tier-0; it must be dedicated to the tenant before assignment") + private Long publicVlanId; + + public Long getZoneId() { + return zoneId; + } + + public String getTier0Gateway() { + return tier0Gateway; + } + + public String getEdgeCluster() { + return edgeCluster; + } + + public String getParentTier0Gateway() { + return parentTier0Gateway; + } + + public Long getPublicVlanId() { + return publicVlanId; + } + + @Override + public void execute() throws ServerApiException { + NsxVrfGatewayResponse response = nsxProviderService.addNsxVrfGateway(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AssignNsxVrfGatewayCmd.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AssignNsxVrfGatewayCmd.java new file mode 100644 index 000000000000..a55422faec4f --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/AssignNsxVrfGatewayCmd.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.service.NsxProviderService; + +/** + * Claims a staged tier-0 gateway for a tenant. Exactly one of accountid or domainid must + * be given: that choice is what {@code nsx.vrf.scope} matches against when a tenant's + * tier-1 gateways are created. + */ +@APICommand(name = AssignNsxVrfGatewayCmd.APINAME, description = "Assigns a registered NSX VRF gateway to an account or a domain", + responseObject = NsxVrfGatewayResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, since = "4.23.0", authorized = {RoleType.Admin}) +public class AssignNsxVrfGatewayCmd extends BaseCmd { + public static final String APINAME = "assignNsxVrfGateway"; + + @Inject + NsxProviderService nsxProviderService; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = NsxVrfGatewayResponse.class, + required = true, description = "the ID of the NSX VRF gateway to assign") + private Long id; + + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "the ID of the account to assign the gateway to") + private Long accountId; + + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "the ID of the domain to assign the gateway to; every account in the domain, and in its " + + "sub-domains, resolves to this gateway unless it has one of its own") + private Long domainId; + + public Long getId() { + return id; + } + + public Long getAccountId() { + return accountId; + } + + public Long getDomainId() { + return domainId; + } + + @Override + public void execute() throws ServerApiException { + NsxVrfGatewayResponse response = nsxProviderService.assignNsxVrfGateway(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/DeleteNsxVrfGatewayCmd.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/DeleteNsxVrfGatewayCmd.java new file mode 100644 index 000000000000..e78c8177bbd0 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/DeleteNsxVrfGatewayCmd.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.service.NsxProviderService; + +/** + * Deregisters a tier-0 gateway from CloudStack. The gateway itself is left untouched in + * NSX, along with its uplinks and BGP peerings — this only removes CloudStack's record. + */ +@APICommand(name = DeleteNsxVrfGatewayCmd.APINAME, description = "Removes CloudStack's registration of an NSX VRF gateway. The gateway itself is not deleted from NSX", + responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, since = "4.23.0", authorized = {RoleType.Admin}) +public class DeleteNsxVrfGatewayCmd extends BaseCmd { + public static final String APINAME = "deleteNsxVrfGateway"; + + @Inject + NsxProviderService nsxProviderService; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = NsxVrfGatewayResponse.class, + required = true, description = "the ID of the NSX VRF gateway registration to remove") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() throws ServerApiException { + boolean result = nsxProviderService.deleteNsxVrfGateway(getId()); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ListNsxVrfGatewaysCmd.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ListNsxVrfGatewaysCmd.java new file mode 100644 index 000000000000..593ec3f8afe3 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ListNsxVrfGatewaysCmd.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.service.NsxProviderService; + +@APICommand(name = ListNsxVrfGatewaysCmd.APINAME, description = "Lists the NSX VRF gateways registered with CloudStack", + responseObject = NsxVrfGatewayResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, since = "4.23.0", authorized = {RoleType.Admin}) +public class ListNsxVrfGatewaysCmd extends BaseListCmd { + public static final String APINAME = "listNsxVrfGateways"; + + @Inject + NsxProviderService nsxProviderService; + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "list gateways in this zone only") + private Long zoneId; + + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "list the gateway assigned to this account only") + private Long accountId; + + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "list the gateway assigned to this domain only") + private Long domainId; + + @Parameter(name = ApiConstants.ALLOCATED_ONLY, type = CommandType.BOOLEAN, + description = "when true, list only gateways already assigned to a tenant; when false, only unassigned ones") + private Boolean allocatedOnly; + + public Long getZoneId() { + return zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public Long getDomainId() { + return domainId; + } + + public Boolean getAllocatedOnly() { + return allocatedOnly; + } + + @Override + public void execute() throws ServerApiException { + List gateways = nsxProviderService.listNsxVrfGateways(this); + ListResponse response = new ListResponse<>(); + response.setResponses(gateways, gateways.size()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ReleaseNsxVrfGatewayCmd.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ReleaseNsxVrfGatewayCmd.java new file mode 100644 index 000000000000..81ebe5dba687 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/command/ReleaseNsxVrfGatewayCmd.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.service.NsxProviderService; + +/** + * Returns a tier-0 gateway to the unassigned pool. Refused while the tenant still has + * networks attached to it, since detaching a live tier-1 from its tier-0 is not something + * CloudStack can do without disrupting the tenant. + */ +@APICommand(name = ReleaseNsxVrfGatewayCmd.APINAME, description = "Releases an NSX VRF gateway from its account or domain, returning it to the pool", + responseObject = NsxVrfGatewayResponse.class, requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, since = "4.23.0", authorized = {RoleType.Admin}) +public class ReleaseNsxVrfGatewayCmd extends BaseCmd { + public static final String APINAME = "releaseNsxVrfGateway"; + + @Inject + NsxProviderService nsxProviderService; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = NsxVrfGatewayResponse.class, + required = true, description = "the ID of the NSX VRF gateway to release") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() throws ServerApiException { + NsxVrfGatewayResponse response = nsxProviderService.releaseNsxVrfGateway(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/response/NsxVrfGatewayResponse.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/response/NsxVrfGatewayResponse.java new file mode 100644 index 000000000000..3e3cde7d8aa1 --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/api/response/NsxVrfGatewayResponse.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = {NsxVrfGatewayVO.class}) +public class NsxVrfGatewayResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the NSX VRF gateway") + private String id; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "ID of the zone the VRF gateway belongs to") + private String zoneId; + + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "Name of the zone the VRF gateway belongs to") + private String zoneName; + + @SerializedName(ApiConstants.TIER0_GATEWAY) + @Param(description = "Name of the VRF, or dedicated, tier-0 gateway as it exists in NSX") + private String tier0Gateway; + + @SerializedName(ApiConstants.EDGE_CLUSTER) + @Param(description = "Name of the edge cluster this tier-0 gateway lives on") + private String edgeCluster; + + @SerializedName(ApiConstants.PARENT_TIER0_GATEWAY) + @Param(description = "Parent tier-0 gateway for a VRF gateway; empty for a dedicated tier-0") + private String parentTier0Gateway; + + @SerializedName(ApiConstants.SCOPE) + @Param(description = "Whether the gateway is assigned per ACCOUNT or per DOMAIN; empty while unassigned") + private String scope; + + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the account the gateway is assigned to") + private String accountId; + + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the account the gateway is assigned to") + private String accountName; + + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "ID of the domain the gateway is assigned to") + private String domainId; + + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "Name of the domain the gateway is assigned to") + private String domainName; + + @SerializedName(ApiConstants.ALLOCATED) + @Param(description = "Whether the gateway has been assigned to a tenant") + private boolean allocated; + + @SerializedName(ApiConstants.VLAN_ID) + @Param(description = "ID of the public IP range advertised by this tier-0") + private String publicVlanId; + + public NsxVrfGatewayResponse() { + setObjectName("nsxvrfgateway"); + } + + public void setId(String id) { + this.id = id; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + public void setTier0Gateway(String tier0Gateway) { + this.tier0Gateway = tier0Gateway; + } + + public void setEdgeCluster(String edgeCluster) { + this.edgeCluster = edgeCluster; + } + + public void setParentTier0Gateway(String parentTier0Gateway) { + this.parentTier0Gateway = parentTier0Gateway; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public void setAccountId(String accountId) { + this.accountId = accountId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + public void setAllocated(boolean allocated) { + this.allocated = allocated; + } + + public void setPublicVlanId(String publicVlanId) { + this.publicVlanId = publicVlanId; + } + + public String getId() { + return id; + } + + public String getTier0Gateway() { + return tier0Gateway; + } + + public String getEdgeCluster() { + return edgeCluster; + } + + public String getScope() { + return scope; + } + + public boolean isAllocated() { + return allocated; + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java index 78a9363a5e49..b7b48be62413 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java @@ -28,12 +28,14 @@ import com.cloud.host.Host; import com.cloud.network.Network; import com.cloud.resource.ServerResource; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.vmware.nsx.model.TransportZone; import com.vmware.nsx.model.TransportZoneListResult; import com.vmware.nsx_policy.model.Segment; import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.NsxVrfGatewayValidationAnswer; import org.apache.cloudstack.StartupNsxCommand; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; import org.apache.cloudstack.agent.api.CreateNsxDistributedFirewallRulesCommand; @@ -42,15 +44,23 @@ import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnGatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnGatewayCommand; +import org.apache.cloudstack.agent.api.GetNsxVpnSessionStatusCommand; +import org.apache.cloudstack.agent.api.UpdateNsxVpnConnectionStateCommand; +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; import org.apache.cloudstack.service.NsxApiClient; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -59,9 +69,21 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class NsxResource implements ServerResource { + + private static final int VPN_TIER1_LOCK_STRIPES = 256; + private static final Object[] VPN_TIER1_LOCKS = createVpnTier1Locks(); + + private static Object[] createVpnTier1Locks() { + Object[] locks = new Object[VPN_TIER1_LOCK_STRIPES]; + for (int i = 0; i < locks.length; i++) { + locks[i] = new Object(); + } + return locks; + } protected Logger logger = LogManager.getLogger(getClass()); private static final String DHCP_RELAY_CONFIGS_PATH_PREFIX = "/infra/dhcp-relay-configs"; @@ -106,6 +128,8 @@ public Answer executeRequest(Command cmd) { return executeRequest((ReadyCommand) cmd); } else if (cmd instanceof CheckHealthCommand) { return executeRequest((CheckHealthCommand) cmd); + } else if (cmd instanceof ValidateNsxVrfGatewayCommand) { + return executeRequest((ValidateNsxVrfGatewayCommand) cmd); } else if (cmd instanceof DeleteNsxTier1GatewayCommand) { return executeRequest((DeleteNsxTier1GatewayCommand) cmd); } else if (cmd instanceof DeleteNsxSegmentCommand) { @@ -132,6 +156,18 @@ public Answer executeRequest(Command cmd) { return executeRequest((DeleteNsxDistributedFirewallRulesCommand) cmd); } else if (cmd instanceof CreateNsxDistributedFirewallRulesCommand) { return executeRequest((CreateNsxDistributedFirewallRulesCommand) cmd); + } else if (cmd instanceof CreateNsxVpnGatewayCommand) { + return executeRequest((CreateNsxVpnGatewayCommand) cmd); + } else if (cmd instanceof DeleteNsxVpnGatewayCommand) { + return executeRequest((DeleteNsxVpnGatewayCommand) cmd); + } else if (cmd instanceof CreateNsxVpnConnectionCommand) { + return executeRequest((CreateNsxVpnConnectionCommand) cmd); + } else if (cmd instanceof DeleteNsxVpnConnectionCommand) { + return executeRequest((DeleteNsxVpnConnectionCommand) cmd); + } else if (cmd instanceof GetNsxVpnSessionStatusCommand) { + return executeRequest((GetNsxVpnSessionStatusCommand) cmd); + } else if (cmd instanceof UpdateNsxVpnConnectionStateCommand) { + return executeRequest((UpdateNsxVpnConnectionStateCommand) cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -301,11 +337,28 @@ private Answer executeRequest(CheckHealthCommand cmd) { return new CheckHealthAnswer(cmd, nsxApiClient.isNsxControllerActive()); } + private Answer executeRequest(ValidateNsxVrfGatewayCommand cmd) { + try { + NsxApiClient.VrfGatewayValidation validation = nsxApiClient.validateVrfGateway( + cmd.getTier0Gateway(), cmd.getExpectedParentTier0(), cmd.getExpectedEdgeCluster()); + return new NsxVrfGatewayValidationAnswer(cmd, true, null, + validation.getEdgeClusterPath(), validation.getParentTier0Path()); + } catch (CloudRuntimeException e) { + return new NsxVrfGatewayValidationAnswer(cmd, e); + } + } + private Answer executeRequest(CreateNsxTier1GatewayCommand cmd) { String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc()); boolean sourceNatEnabled = cmd.isSourceNatEnabled(); + String targetTier0 = StringUtils.defaultIfBlank(cmd.getTier0Gateway(), tier0Gateway); + String targetEdgeCluster = StringUtils.defaultIfBlank(cmd.getEdgeCluster(), edgeCluster); + if (!targetTier0.equals(tier0Gateway) || !targetEdgeCluster.equals(edgeCluster)) { + logger.debug("Creating tier 1 gateway {} on tier 0 {} (edge cluster {}) instead of the zone default {} ({})", + tier1GatewayName, targetTier0, targetEdgeCluster, tier0Gateway, edgeCluster); + } try { - nsxApiClient.createTier1Gateway(tier1GatewayName, tier0Gateway, edgeCluster, sourceNatEnabled); + nsxApiClient.createTier1Gateway(tier1GatewayName, targetTier0, targetEdgeCluster, sourceNatEnabled); return new NsxAnswer(cmd, true, ""); } catch (CloudRuntimeException e) { String msg = String.format("Cannot create tier 1 gateway %s (%s: %s): %s", tier1GatewayName, @@ -318,11 +371,14 @@ private Answer executeRequest(CreateNsxTier1GatewayCommand cmd) { private Answer executeRequest(DeleteNsxTier1GatewayCommand cmd) { String tier1Id = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc()); String lbName = NsxControllerUtils.getLoadBalancerName(tier1Id); - try { - nsxApiClient.deleteLoadBalancer(lbName); - nsxApiClient.deleteTier1Gateway(tier1Id); - } catch (Exception e) { - return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + Object lock = getVpnTier1Lock(tier1Id); + synchronized (lock) { + try { + nsxApiClient.deleteLoadBalancer(lbName); + nsxApiClient.deleteTier1Gateway(tier1Id); + } catch (Exception e) { + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } } return new NsxAnswer(cmd, true, null); } @@ -351,7 +407,8 @@ private Answer executeRequest(CreateNsxSegmentCommand cmd) { boolean isResourceVpc = !Objects.isNull(cmd.getVpcId()); String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), networkResourceId, isResourceVpc); - nsxApiClient.createSegment(segmentName, tier1GatewayName, gatewayAddress, enforcementPointPath, transportZones); + nsxApiClient.createSegment(segmentName, tier1GatewayName, gatewayAddress, enforcementPointPath, transportZones, + cmd.getIpDiscoveryProfileId(), cmd.getMacDiscoveryProfileId(), cmd.getSegmentSecurityProfileId()); nsxApiClient.createGroupForSegment(segmentName); } catch (Exception e) { logger.error(String.format("Failed to create network: %s", cmd.getNetworkName())); @@ -488,6 +545,165 @@ private NsxAnswer executeRequest(DeleteNsxDistributedFirewallRulesCommand cmd) { return new NsxAnswer(cmd, true, null); } + private NsxAnswer executeRequest(CreateNsxVpnGatewayCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + Object lock = getVpnTier1Lock(tier1GatewayName); + synchronized (lock) { + final boolean vpnServiceExisted; + try { + vpnServiceExisted = nsxApiClient.isVpnServicePresent(tier1GatewayName); + } catch (Exception e) { + logger.error(String.format("Failed to check the existing NSX VPN service on tier-1 gateway %s before creation: %s", + tier1GatewayName, e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + if (vpnServiceExisted) { + NsxAnswer answer = new NsxAnswer(cmd, new CloudRuntimeException(String.format( + "An NSX VPN service already exists on tier-1 gateway %s; refusing to adopt or overwrite it", + tier1GatewayName))); + answer.setObjectExists(true); + return answer; + } + try { + nsxApiClient.createVpnService(tier1GatewayName, cmd.getLocalEndpointIp()); + } catch (Exception e) { + boolean endpointMayBeInUse = false; + try { + nsxApiClient.deleteVpnService(tier1GatewayName); + } catch (Exception rollbackException) { + endpointMayBeInUse = true; + logger.warn("Failed to roll back the newly-created NSX VPN service on tier-1 gateway {} after creation failed: {}", + tier1GatewayName, rollbackException.getMessage()); + } + logger.error(String.format("Failed to create the NSX VPN service on tier-1 gateway %s for VPC %s: %s", + tier1GatewayName, cmd.getVpcName(), e.getMessage())); + NsxAnswer answer = new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + answer.setEndpointMayBeInUse(endpointMayBeInUse); + return answer; + } + } + NsxAnswer answer = new NsxAnswer(cmd, true, null); + answer.setEndpointMayBeInUse(true); + return answer; + } + + private NsxAnswer executeRequest(DeleteNsxVpnGatewayCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + Object lock = getVpnTier1Lock(tier1GatewayName); + synchronized (lock) { + try { + nsxApiClient.deleteVpnService(tier1GatewayName); + } catch (Exception e) { + logger.error(String.format("Failed to delete the NSX VPN service on tier-1 gateway %s for VPC %s: %s", + tier1GatewayName, cmd.getVpcName(), e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(CreateNsxVpnConnectionCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + Object lock = getVpnTier1Lock(tier1GatewayName); + synchronized (lock) { + NsxApiClient.VpnSessionProvisioningResult provisioningResult = null; + try { + // The requested VTI /30 is derived from the connection id. Fail closed when another + // session already owns its local address; silently choosing a different pair would make + // the peer route advertised by CloudStack disagree with the pair installed in NSX. + Set inUseVtiIps = nsxApiClient.getRouteBasedVpnSessionLocalVtiIps(tier1GatewayName, cmd.getConnectionUuid()); + if (inUseVtiIps.contains(cmd.getVtiLocalIp())) { + throw new CloudRuntimeException(String.format( + "The deterministic VTI address %s for VPN connection %s is already in use on tier-1 gateway %s", + cmd.getVtiLocalIp(), cmd.getConnectionUuid(), tier1GatewayName)); + } + Pair vtiAddresses = new Pair<>(cmd.getVtiLocalIp(), cmd.getVtiPeerIp()); + provisioningResult = nsxApiClient.createRouteBasedVpnSession(tier1GatewayName, cmd.getConnectionUuid(), cmd.getPeerAddress(), + cmd.getPsk(), cmd.getIkePolicy(), cmd.getEspPolicy(), cmd.getIkeLifetime(), cmd.getEspLifetime(), + cmd.isDpdEnabled(), cmd.getIkeVersion(), cmd.isPassive(), vtiAddresses.first(), cmd.getVtiPrefixLength()); + nsxApiClient.addVpnConnectionRoutes(tier1GatewayName, cmd.getConnectionUuid(), cmd.getPeerCidrs(), + vtiAddresses.second(), cmd.getVpcCidr()); + // Applied here as well so that VPN gateways created before the exemptions existed, or whose + // tier-1 gained a source NAT rule afterwards, are corrected without recreating the gateway + nsxApiClient.ensureVpnNatExemptions(tier1GatewayName, cmd.getLocalEndpointIp()); + nsxApiClient.updateVpnConnectionState(tier1GatewayName, cmd.getConnectionUuid(), true); + } catch (Exception e) { + if (provisioningResult == NsxApiClient.VpnSessionProvisioningResult.CREATED) { + try { + nsxApiClient.rollbackVpnConnection(tier1GatewayName, cmd.getConnectionUuid()); + } catch (Exception rollbackException) { + logger.warn("Failed to roll back the partially created NSX VPN connection {}: {}", + cmd.getConnectionUuid(), rollbackException.getMessage()); + } + } else if (provisioningResult == NsxApiClient.VpnSessionProvisioningResult.PREEXISTING) { + try { + nsxApiClient.updateVpnConnectionState(tier1GatewayName, cmd.getConnectionUuid(), false); + } catch (Exception compensationException) { + logger.warn("Failed to disable the pre-existing NSX VPN connection {} after provisioning failed: {}", + cmd.getConnectionUuid(), compensationException.getMessage()); + } + } + logger.error(String.format("Failed to create the NSX VPN connection %s on tier-1 gateway %s for VPC %s: %s", + cmd.getConnectionUuid(), tier1GatewayName, cmd.getVpcName(), e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(DeleteNsxVpnConnectionCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + Object lock = getVpnTier1Lock(tier1GatewayName); + synchronized (lock) { + try { + nsxApiClient.deleteVpnConnection(tier1GatewayName, cmd.getConnectionUuid()); + } catch (Exception e) { + logger.error(String.format("Failed to delete the NSX VPN connection %s on tier-1 gateway %s for VPC %s: %s", + cmd.getConnectionUuid(), tier1GatewayName, cmd.getVpcName(), e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + } + return new NsxAnswer(cmd, true, null); + } + + private NsxAnswer executeRequest(UpdateNsxVpnConnectionStateCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + Object lock = getVpnTier1Lock(tier1GatewayName); + synchronized (lock) { + try { + nsxApiClient.updateVpnConnectionState(tier1GatewayName, cmd.getConnectionUuid(), cmd.isEnabled()); + } catch (Exception e) { + logger.error(String.format("Failed to update the state of the NSX VPN connection %s on tier-1 gateway %s for VPC %s: %s", + cmd.getConnectionUuid(), tier1GatewayName, cmd.getVpcName(), e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + } + return new NsxAnswer(cmd, true, null); + } + + private Object getVpnTier1Lock(String tier1GatewayName) { + return VPN_TIER1_LOCKS[Math.floorMod(tier1GatewayName.hashCode(), VPN_TIER1_LOCKS.length)]; + } + + private NsxAnswer executeRequest(GetNsxVpnSessionStatusCommand cmd) { + String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(cmd.getDomainId(), cmd.getAccountId(), + cmd.getZoneId(), cmd.getVpcId(), true); + String status; + try { + status = nsxApiClient.getVpnSessionStatus(tier1GatewayName, cmd.getConnectionUuid()); + } catch (Exception e) { + logger.error(String.format("Failed to get the status of the NSX VPN connection %s on tier-1 gateway %s for VPC %s: %s", + cmd.getConnectionUuid(), tier1GatewayName, cmd.getVpcName(), e.getMessage())); + return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); + } + return new NsxAnswer(cmd, true, status); + } + @Override public boolean start() { return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 4d78f2a0ab26..0722090c2b92 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -25,30 +25,59 @@ import com.vmware.nsx.model.ControllerClusterStatus; import com.vmware.nsx.model.TransportZone; import com.vmware.nsx.model.TransportZoneListResult; +import com.vmware.nsx_policy.Infra; import com.vmware.nsx_policy.infra.DhcpRelayConfigs; +import com.vmware.nsx_policy.infra.IpsecVpnDpdProfiles; +import com.vmware.nsx_policy.infra.IpsecVpnIkeProfiles; +import com.vmware.nsx_policy.infra.IpsecVpnTunnelProfiles; +import com.vmware.nsx_policy.infra.IpDiscoveryProfiles; import com.vmware.nsx_policy.infra.LbAppProfiles; import com.vmware.nsx_policy.infra.LbMonitorProfiles; import com.vmware.nsx_policy.infra.LbPools; import com.vmware.nsx_policy.infra.LbServices; import com.vmware.nsx_policy.infra.LbVirtualServers; +import com.vmware.nsx_policy.infra.MacDiscoveryProfiles; +import com.vmware.nsx_policy.infra.SegmentSecurityProfiles; import com.vmware.nsx_policy.infra.Segments; import com.vmware.nsx_policy.infra.Services; import com.vmware.nsx_policy.infra.Sites; import com.vmware.nsx_policy.infra.Tier1s; +import com.vmware.nsx_policy.infra.Tier0s; import com.vmware.nsx_policy.infra.domains.Groups; import com.vmware.nsx_policy.infra.domains.SecurityPolicies; import com.vmware.nsx_policy.infra.domains.groups.members.SegmentPorts; import com.vmware.nsx_policy.infra.domains.security_policies.Rules; import com.vmware.nsx_policy.infra.sites.EnforcementPoints; import com.vmware.nsx_policy.infra.tier_0s.LocaleServices; +import com.vmware.nsx_policy.infra.tier_0s.locale_services.Interfaces; +import com.vmware.nsx_policy.infra.tier_1s.IpsecVpnServices; +import com.vmware.nsx_policy.infra.tier_1s.ipsec_vpn_services.LocalEndpoints; +import com.vmware.nsx_policy.infra.tier_1s.ipsec_vpn_services.Sessions; +import com.vmware.nsx_policy.infra.tier_1s.ipsec_vpn_services.sessions.DetailedStatus; import com.vmware.nsx_policy.infra.tier_1s.nat.NatRules; +import com.vmware.nsx_policy.model.AggregateIPSecVpnSessionStatus; import com.vmware.nsx_policy.model.ApiError; +import com.vmware.nsx_policy.model.ChildSegment; +import com.vmware.nsx_policy.model.ChildSegmentDiscoveryProfileBindingMap; +import com.vmware.nsx_policy.model.ChildSegmentSecurityProfileBindingMap; import com.vmware.nsx_policy.model.DhcpRelayConfig; import com.vmware.nsx_policy.model.EnforcementPoint; import com.vmware.nsx_policy.model.EnforcementPointListResult; import com.vmware.nsx_policy.model.Group; import com.vmware.nsx_policy.model.GroupListResult; import com.vmware.nsx_policy.model.ICMPTypeServiceEntry; +import com.vmware.nsx_policy.model.IPSecVpnDpdProfile; +import com.vmware.nsx_policy.model.IPSecVpnIkeProfile; +import com.vmware.nsx_policy.model.IPSecVpnLocalEndpoint; +import com.vmware.nsx_policy.model.IPSecVpnLocalEndpointListResult; +import com.vmware.nsx_policy.model.IPSecVpnService; +import com.vmware.nsx_policy.model.IPSecVpnSession; +import com.vmware.nsx_policy.model.IPSecVpnServiceListResult; +import com.vmware.nsx_policy.model.IPSecVpnSessionListResult; +import com.vmware.nsx_policy.model.IPSecVpnSessionStatusNsxt; +import com.vmware.nsx_policy.model.IPSecVpnTunnelInterface; +import com.vmware.nsx_policy.model.IPSecVpnTunnelProfile; +import com.vmware.nsx_policy.model.IPDiscoveryProfile; import com.vmware.nsx_policy.model.L4PortSetServiceEntry; import com.vmware.nsx_policy.model.LBAppProfileListResult; import com.vmware.nsx_policy.model.LBIcmpMonitorProfile; @@ -61,18 +90,31 @@ import com.vmware.nsx_policy.model.LBVirtualServer; import com.vmware.nsx_policy.model.LBVirtualServerListResult; import com.vmware.nsx_policy.model.LocaleServicesListResult; +import com.vmware.nsx_policy.model.MacDiscoveryProfile; import com.vmware.nsx_policy.model.PathExpression; import com.vmware.nsx_policy.model.PolicyGroupMembersListResult; +import com.vmware.nsx_policy.model.PolicyBgpNeighborStatus; +import com.vmware.nsx_policy.model.PolicyBgpNeighborsStatusListResult; import com.vmware.nsx_policy.model.PolicyNatRule; import com.vmware.nsx_policy.model.PolicyNatRuleListResult; import com.vmware.nsx_policy.model.PolicyGroupMemberDetails; +import com.vmware.nsx_policy.model.RouteBasedIPSecVpnSession; +import com.vmware.nsx_policy.model.RouterNexthop; import com.vmware.nsx_policy.model.Rule; import com.vmware.nsx_policy.model.SecurityPolicy; import com.vmware.nsx_policy.model.Segment; +import com.vmware.nsx_policy.model.SegmentDiscoveryProfileBindingMap; +import com.vmware.nsx_policy.model.SegmentSecurityProfile; +import com.vmware.nsx_policy.model.SegmentSecurityProfileBindingMap; import com.vmware.nsx_policy.model.SegmentSubnet; import com.vmware.nsx_policy.model.ServiceListResult; import com.vmware.nsx_policy.model.Site; +import com.vmware.nsx_policy.model.StaticRoutesListResult; +import com.vmware.nsx_policy.model.Tag; import com.vmware.nsx_policy.model.Tier1; +import com.vmware.nsx_policy.model.Tier0; +import com.vmware.nsx_policy.model.Tier0InterfaceListResult; +import com.vmware.nsx_policy.model.TunnelInterfaceIPSubnet; import com.vmware.vapi.bindings.Service; import com.vmware.vapi.bindings.Structure; import com.vmware.vapi.bindings.StubConfiguration; @@ -89,12 +131,15 @@ import org.apache.cloudstack.resource.NsxLoadBalancerMember; import org.apache.cloudstack.resource.NsxNetworkRule; import org.apache.cloudstack.utils.NsxControllerUtils; +import org.apache.cloudstack.utils.NsxVpnCryptoUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.commons.lang3.BooleanUtils; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Objects; @@ -102,6 +147,7 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.stream.Collectors.toSet; @@ -114,6 +160,17 @@ import static org.apache.cloudstack.utils.NsxControllerUtils.getLoadBalancerAlgorithm; import static org.apache.cloudstack.utils.NsxControllerUtils.getActiveMonitorProfileName; import static org.apache.cloudstack.utils.NsxControllerUtils.getTier1GatewayName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnDpdProfileName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnEspProfileName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnIkeProfileName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnLocalEndpointName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnLocalEndpointNoSnatRuleName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnNoSnatRuleName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnNoSnatRuleNamePrefix; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnServiceName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnSessionName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnStaticRouteName; +import static org.apache.cloudstack.utils.NsxControllerUtils.getVpnStaticRouteNamePrefix; public class NsxApiClient { @@ -128,6 +185,11 @@ public class NsxApiClient { private static final String TIER_1_RESOURCE_TYPE = "Tier1"; private static final String TIER_1_LOCALE_SERVICE_ID = "default"; private static final String SEGMENT_RESOURCE_TYPE = "Segment"; + private static final String INFRA_RESOURCE_TYPE = "Infra"; + private static final String SEGMENT_DISCOVERY_PROFILE_BINDING_RESOURCE_TYPE = "SegmentDiscoveryProfileBindingMap"; + private static final String SEGMENT_SECURITY_PROFILE_BINDING_RESOURCE_TYPE = "SegmentSecurityProfileBindingMap"; + private static final String SEGMENT_DISCOVERY_PROFILE_BINDING_ID = "cloudstack-discovery-profile-binding"; + private static final String SEGMENT_SECURITY_PROFILE_BINDING_ID = "cloudstack-security-profile-binding"; private static final String TIER_0_GATEWAY_PATH_PREFIX = "/infra/tier-0s/"; private static final String TIER_1_GATEWAY_PATH_PREFIX = "/infra/tier-1s/"; protected static final String SEGMENTS_PATH = "/infra/segments"; @@ -138,6 +200,32 @@ public class NsxApiClient { protected static final String TCP_MONITOR_PROFILE = "LBTcpMonitorProfile"; protected static final String ICMP_MONITOR_PROFILE = "LBIcmpMonitorProfile"; protected static final String NAT_ID = "USER"; + protected static final String IPSEC_VPN_IKE_PROFILES_PATH_PREFIX = "/infra/ipsec-vpn-ike-profiles/"; + protected static final String IPSEC_VPN_TUNNEL_PROFILES_PATH_PREFIX = "/infra/ipsec-vpn-tunnel-profiles/"; + protected static final String IPSEC_VPN_DPD_PROFILES_PATH_PREFIX = "/infra/ipsec-vpn-dpd-profiles/"; + // NSX resolves NAT rules scoped to a VTI only for a tunnel interface with this exact name (KB 435087) + protected static final String VPN_DEFAULT_TUNNEL_INTERFACE_NAME = "default-tunnel-interface"; + // NSX purges objects marked for deletion in a cycle that it documents as taking up to 5 minutes, + // and rejects recreating an object under the same path until then + protected static final int VPN_MARKED_FOR_DELETION_RETRIES = 24; + protected static final int VPN_MARKED_FOR_DELETION_RETRY_INTERVAL_SECS = 15; + // In on demand mode the probe interval is the idle time before a probe is sent, which NSX limits + // to 1-10 seconds (the 3-360 second range only applies to periodic probing) + protected static final long VPN_DPD_PROBE_INTERVAL_SECS = 10L; + protected static final long VPN_DPD_RETRY_COUNT = 10L; + // NSX evaluates NAT rules by ascending sequence number, so the traffic the VPN exempts from source + // NAT has to be matched before the catch all source NAT rule of the VPC + protected static final long VPN_NO_SNAT_SEQUENCE_NUMBER = 100L; + protected static final long CATCH_ALL_NAT_SEQUENCE_NUMBER = 1000L; + protected static final String VPN_ORIGINAL_SNAT_SEQUENCE_TAG_SCOPE = "cloudstack-vpn-original-snat-sequence"; + protected static final int NSX_MAX_TAGS = 30; + protected static final String VPN_SESSION_STATUS_UNKNOWN = "UNKNOWN"; + protected static final String VPN_SESSION_STATUS_NOT_FOUND = "NOT_FOUND"; + + public enum VpnSessionProvisioningResult { + CREATED, + PREEXISTING + } private enum PoolAllocation { ROUTING, LB_SMALL, LB_MEDIUM, LB_LARGE, LB_XLARGE } @@ -151,7 +239,7 @@ private enum TransportType { OVERLAY, VLAN } private enum NatId { USER, INTERNAL, DEFAULT } - private enum NatAction {SNAT, DNAT, REFLEXIVE} + private enum NatAction {SNAT, DNAT, REFLEXIVE, NO_SNAT} private enum FirewallMatch { MATCH_INTERNAL_ADDRESS, @@ -242,6 +330,144 @@ public void createTier1NatRule(String tier1GatewayName, String natId, String nat natRulesService.patch(tier1GatewayName, natId, natRuleId, natPolicy); } + /** + * The IKE traffic the gateway originates from the local endpoint must keep that address as its + * source: the catch all source NAT rule would otherwise rewrite it to the VPC source NAT IP and the + * peer, which only knows the local endpoint address, would ignore the packets. Applied whenever a + * VPN service or connection is created so that gateways predating this also get the exemption. + */ + public void ensureVpnNatExemptions(String tier1GatewayName, String localEndpointIp) { + demoteCatchAllSourceNatRule(tier1GatewayName); + String localEndpointNoSnatRuleName = getVpnLocalEndpointNoSnatRuleName(getVpnServiceName(tier1GatewayName)); + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + PolicyNatRule localEndpointNoSnatRule = new PolicyNatRule.Builder() + .setId(localEndpointNoSnatRuleName) + .setDisplayName(localEndpointNoSnatRuleName) + .setAction(NatAction.NO_SNAT.name()) + .setSourceNetwork(localEndpointIp) + .setSequenceNumber(VPN_NO_SNAT_SEQUENCE_NUMBER) + .setEnabled(true) + .build(); + natService.patch(tier1GatewayName, NatId.USER.name(), localEndpointNoSnatRuleName, localEndpointNoSnatRule); + } + + /** Existing CloudStack source NAT must be evaluated after the VPN no-SNAT rules. */ + private void demoteCatchAllSourceNatRule(String tier1GatewayName) { + NatRules natRulesService = (NatRules) nsxService.apply(NatRules.class); + try { + String ruleId = getCloudStackSourceNatRuleId(tier1GatewayName); + PolicyNatRule natRule = natRulesService.get(tier1GatewayName, NatId.USER.name(), ruleId); + if (!isCatchAllSourceNatRule(natRule)) { + return; + } + List tags = copyNatRuleTags(natRule); + Tag originalSequenceTag = findOriginalSnatSequenceTag(tags); + if (originalSequenceTag == null) { + if (tags.size() >= NSX_MAX_TAGS) { + throw new CloudRuntimeException(String.format( + "Cannot preserve the source NAT rule sequence of tier-1 gateway %s because the rule already has the maximum number of tags", + tier1GatewayName)); + } + long originalSequence = natRule.getSequenceNumber() == null ? 0L : natRule.getSequenceNumber(); + tags.add(new Tag.Builder() + .setScope(VPN_ORIGINAL_SNAT_SEQUENCE_TAG_SCOPE) + .setTag(String.valueOf(originalSequence)) + .build()); + } + logger.debug("Moving CloudStack source NAT rule {} on tier-1 gateway {} behind VPN no-SNAT rules", + ruleId, tier1GatewayName); + natRulesService.patch(tier1GatewayName, NatId.USER.name(), ruleId, + copyNatRule(natRule, CATCH_ALL_NAT_SEQUENCE_NUMBER, tags)); + } catch (NotFound e) { + logger.debug("CloudStack source NAT rule is absent on tier-1 gateway {}; no VPN NAT ordering change is required", + tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to order the source NAT rules of tier-1 gateway %s before creating the NSX VPN exemptions: %s", + tier1GatewayName, ae.getErrorMessage()), error); + } + } + + void restoreSourceNatRuleSequence(String tier1GatewayName) { + NatRules natRulesService = (NatRules) nsxService.apply(NatRules.class); + try { + String ruleId = getCloudStackSourceNatRuleId(tier1GatewayName); + PolicyNatRule natRule = natRulesService.get(tier1GatewayName, NatId.USER.name(), ruleId); + List tags = copyNatRuleTags(natRule); + Tag originalSequenceTag = findOriginalSnatSequenceTag(tags); + if (originalSequenceTag == null) { + return; + } + long originalSequence; + try { + originalSequence = Long.parseLong(originalSequenceTag.getTag()); + } catch (NumberFormatException e) { + throw new CloudRuntimeException(String.format( + "Invalid saved source NAT sequence '%s' on tier-1 gateway %s", + originalSequenceTag.getTag(), tier1GatewayName), e); + } + tags.remove(originalSequenceTag); + natRulesService.patch(tier1GatewayName, NatId.USER.name(), ruleId, + copyNatRule(natRule, originalSequence, tags)); + } catch (NotFound e) { + logger.debug("CloudStack source NAT rule is absent on tier-1 gateway {}; no sequence restoration is required", + tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to restore the source NAT rule order of tier-1 gateway %s after deleting the NSX VPN gateway: %s", + tier1GatewayName, ae.getErrorMessage()), error); + } + } + + private String getCloudStackSourceNatRuleId(String tier1GatewayName) { + return tier1GatewayName + "-NAT"; + } + + private List copyNatRuleTags(PolicyNatRule natRule) { + return natRule.getTags() == null ? new ArrayList<>() : new ArrayList<>(natRule.getTags()); + } + + private Tag findOriginalSnatSequenceTag(List tags) { + return tags.stream() + .filter(tag -> VPN_ORIGINAL_SNAT_SEQUENCE_TAG_SCOPE.equals(tag.getScope())) + .findFirst() + .orElse(null); + } + + private PolicyNatRule copyNatRule(PolicyNatRule natRule, long sequenceNumber, List tags) { + return new PolicyNatRule.Builder() + .setId(natRule.getId()) + .setDisplayName(natRule.getDisplayName()) + .setDescription(natRule.getDescription()) + .setAction(natRule.getAction()) + .setTranslatedNetwork(natRule.getTranslatedNetwork()) + .setTranslatedPorts(natRule.getTranslatedPorts()) + .setSourceNetwork(natRule.getSourceNetwork()) + .setDestinationNetwork(natRule.getDestinationNetwork()) + .setService(natRule.getService()) + .setScope(natRule.getScope()) + .setFirewallMatch(natRule.getFirewallMatch()) + .setPolicyBasedVpnMode(natRule.getPolicyBasedVpnMode()) + .setLogging(natRule.getLogging()) + .setEnabled(natRule.getEnabled()) + .setTags(tags) + .setSequenceNumber(sequenceNumber) + .build(); + } + + private boolean isCatchAllSourceNatRule(PolicyNatRule natRule) { + return NatAction.SNAT.name().equals(natRule.getAction()) + && isAnyNatMatch(natRule.getSourceNetwork()) + && isAnyNatMatch(natRule.getDestinationNetwork()); + } + + private boolean isAnyNatMatch(String network) { + return network == null || network.isBlank() || "ANY".equalsIgnoreCase(network) + || "0.0.0.0/0".equals(network) || "::/0".equals(network); + } + public void createDhcpRelayConfig(String dhcpRelayConfigName, List addresses) { try { DhcpRelayConfigs service = (DhcpRelayConfigs) nsxService.apply(DhcpRelayConfigs.class); @@ -293,24 +519,137 @@ private Tier1 getTier1Gateway(String tier1GatewayId) { return null; } - private Optional findTier0LocalServices(String tier0Gateway) { + private Optional findTier0LocalServices(String tier0Gateway, + String expectedEdgeCluster) { try { LocaleServices tier0LocaleServices = (LocaleServices) nsxService.apply(LocaleServices.class); - LocaleServicesListResult result = tier0LocaleServices.list(tier0Gateway, null, false, null, 1L, null, null); - return Optional.ofNullable(result.getResults()) - .filter(Predicate.not(List::isEmpty)) - .map(l -> l.get(0)); + LocaleServicesListResult result = tier0LocaleServices.list(tier0Gateway, null, false, null, 1000L, null, null); + List localeServices = Optional.ofNullable(result.getResults()) + .orElseGet(List::of); + if (localeServices.isEmpty()) { + return Optional.empty(); + } + if (StringUtils.isBlank(expectedEdgeCluster)) { + if (localeServices.size() != 1) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s has multiple locale services; specify the expected edge cluster", + tier0Gateway)); + } + return Optional.of(localeServices.get(0)); + } + List matchingLocaleServices = localeServices.stream() + .filter(localeService -> expectedEdgeCluster.equals(localeService.getEdgeClusterPath()) + || StringUtils.endsWith(localeService.getEdgeClusterPath(), "/" + expectedEdgeCluster)) + .toList(); + if (matchingLocaleServices.size() != 1) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s has %d locale services on expected edge cluster %s", + tier0Gateway, matchingLocaleServices.size(), expectedEdgeCluster)); + } + return Optional.of(matchingLocaleServices.get(0)); + } catch (CloudRuntimeException e) { + throw e; } catch (Exception e) { throw new CloudRuntimeException(String.format("Failed to fetch locale services for tier gateway %s due to %s", tier0Gateway, e.getMessage())); } } + public static class VrfGatewayValidation { + private final String edgeClusterPath; + private final String parentTier0Path; + + VrfGatewayValidation(String edgeClusterPath, String parentTier0Path) { + this.edgeClusterPath = edgeClusterPath; + this.parentTier0Path = parentTier0Path; + } + + public String getEdgeClusterPath() { + return edgeClusterPath; + } + + public String getParentTier0Path() { + return parentTier0Path; + } + } + + public VrfGatewayValidation validateVrfGateway(String tier0Gateway, String expectedParentTier0, + String expectedEdgeCluster) { + try { + Tier0s tier0s = (Tier0s) nsxService.apply(Tier0s.class); + Tier0 tier0 = tier0s.get(tier0Gateway); + if (tier0 == null || Boolean.TRUE.equals(tier0.getMarkedForDelete())) { + throw new CloudRuntimeException(String.format("Tier-0 gateway %s is not available", tier0Gateway)); + } + + String actualParentPath = tier0.getVrfConfig() == null ? null : tier0.getVrfConfig().getTier0Path(); + if (StringUtils.isBlank(expectedParentTier0) != StringUtils.isBlank(actualParentPath)) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s VRF parent does not match the declared gateway type", tier0Gateway)); + } + if (StringUtils.isNotBlank(expectedParentTier0)) { + String expectedParentPath = expectedParentTier0.startsWith(TIER_0_GATEWAY_PATH_PREFIX) + ? expectedParentTier0 : TIER_0_GATEWAY_PATH_PREFIX + expectedParentTier0; + if (!expectedParentPath.equals(actualParentPath)) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s has parent %s, expected %s", + tier0Gateway, actualParentPath, expectedParentPath)); + } + } + + com.vmware.nsx_policy.model.LocaleServices localeService = findTier0LocalServices(tier0Gateway, expectedEdgeCluster) + .orElseThrow(() -> new CloudRuntimeException(String.format( + "Tier-0 gateway %s has no locale service", tier0Gateway))); + String edgeClusterPath = localeService.getEdgeClusterPath(); + if (StringUtils.isBlank(edgeClusterPath)) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s locale service has no edge cluster", tier0Gateway)); + } + if (StringUtils.isNotBlank(expectedEdgeCluster) + && !edgeClusterPath.equals(expectedEdgeCluster) + && !edgeClusterPath.endsWith("/" + expectedEdgeCluster)) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s uses edge cluster %s, expected %s", + tier0Gateway, edgeClusterPath, expectedEdgeCluster)); + } + + Interfaces interfaces = (Interfaces) nsxService.apply(Interfaces.class); + Tier0InterfaceListResult interfaceResult = interfaces.list(tier0Gateway, localeService.getId(), + null, false, null, 1L, null, null); + if (CollectionUtils.isEmpty(interfaceResult.getResults())) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s has no locale-service interface", tier0Gateway)); + } + + com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status statusService = + (com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status) nsxService.apply( + com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status.class); + PolicyBgpNeighborsStatusListResult statuses = statusService.list(tier0Gateway, localeService.getId(), + null, null, null, false, null, null, null, null); + boolean established = !CollectionUtils.isEmpty(statuses.getResults()) && statuses.getResults().stream() + .anyMatch(status -> PolicyBgpNeighborStatus.CONNECTION_STATE_ESTABLISHED.equals(status.getConnectionState())); + if (!established) { + throw new CloudRuntimeException(String.format( + "Tier-0 gateway %s has no established BGP neighbor", tier0Gateway)); + } + return new VrfGatewayValidation(edgeClusterPath, actualParentPath); + } catch (CloudRuntimeException e) { + throw e; + } catch (Error error) { + ApiError apiError = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to validate Tier-0 gateway %s: %s", tier0Gateway, apiError.getErrorMessage())); + } catch (Exception e) { + throw new CloudRuntimeException(String.format( + "Failed to validate Tier-0 gateway %s: %s", tier0Gateway, e.getMessage()), e); + } + } + /** * To instantiate Tier-1 in Edge Cluster */ private void createTier1LocaleServices(String tier1Id, String edgeCluster, String tier0Gateway) { try { - Optional localeServices = findTier0LocalServices(tier0Gateway); + Optional localeServices = findTier0LocalServices(tier0Gateway, edgeCluster); if (localeServices.isEmpty()) { throw new CloudRuntimeException(String.format("Failed to find locale services for tier-0 gateway %s", tier0Gateway)); } @@ -338,8 +677,12 @@ public void createTier1Gateway(String name, String tier0Gateway, String edgeClus String tier0GatewayPath = TIER_0_GATEWAY_PATH_PREFIX + tier0Gateway; Tier1 tier1 = getTier1Gateway(name); if (tier1 != null) { - logger.info("VPC network with name {} exists in NSX zone", name); - return; + if (!tier0GatewayPath.equals(tier1.getTier0Path())) { + throw new CloudRuntimeException(String.format( + "Tier-1 gateway %s already exists under %s, not requested parent %s", + name, tier1.getTier0Path(), tier0GatewayPath)); + } + logger.info("Reconciling existing Tier-1 gateway {} in NSX", name); } List routeAdvertisementTypes = getRouterAdvertisementTypeList(sourceNatEnabled); @@ -373,6 +716,7 @@ public void deleteTier1Gateway(String tier1Id) { logger.warn("The Tier 1 Gateway {} does not exist, cannot be removed", tier1Id); return; } + removeTier1VpnResources(tier1Id); removeTier1GatewayNatRules(tier1Id); localeService.delete(tier1Id, TIER_1_LOCALE_SERVICE_ID); Tier1s tier1service = (Tier1s) nsxService.apply(Tier1s.class); @@ -466,8 +810,16 @@ public TransportZoneListResult getTransportZones() { public void createSegment(String segmentName, String tier1GatewayName, String gatewayAddress, String enforcementPointPath, List transportZones) { + createSegment(segmentName, tier1GatewayName, gatewayAddress, enforcementPointPath, transportZones, null, null, null); + } + + public void createSegment(String segmentName, String tier1GatewayName, String gatewayAddress, String enforcementPointPath, + List transportZones, String ipDiscoveryProfileId, String macDiscoveryProfileId, + String segmentSecurityProfileId) { try { - Segments segmentService = (Segments) nsxService.apply(Segments.class); + String ipDiscoveryProfilePath = getIpDiscoveryProfilePath(ipDiscoveryProfileId); + String macDiscoveryProfilePath = getMacDiscoveryProfilePath(macDiscoveryProfileId); + String segmentSecurityProfilePath = getSegmentSecurityProfilePath(segmentSecurityProfileId); SegmentSubnet subnet = new SegmentSubnet.Builder() .setGatewayAddress(gatewayAddress) .build(); @@ -480,7 +832,24 @@ public void createSegment(String segmentName, String tier1GatewayName, String ga .setSubnets(List.of(subnet)) .setTransportZonePath(enforcementPointPath + "/transport-zones/" + transportZones.get(0).getId()) .build(); - segmentService.patch(segmentName, segment); + List profileBindings = getSegmentProfileBindings(ipDiscoveryProfilePath, macDiscoveryProfilePath, + segmentSecurityProfilePath); + if (profileBindings.isEmpty()) { + Segments segmentService = (Segments) nsxService.apply(Segments.class); + segmentService.patch(segmentName, segment); + } else { + segment.setChildren(profileBindings); + ChildSegment childSegment = new ChildSegment.Builder() + .setId(segmentName) + .setSegment(segment) + .build(); + com.vmware.nsx_policy.model.Infra infra = new com.vmware.nsx_policy.model.Infra.Builder() + .setResourceType(INFRA_RESOURCE_TYPE) + .setChildren(List.of(childSegment)) + .build(); + Infra infraService = (Infra) nsxService.apply(Infra.class); + infraService.patch(infra, false); + } } catch (Error error) { ApiError ae = error.getData()._convertTo(ApiError.class); String msg = String.format("Error creating segment %s: %s", segmentName, ae.getErrorMessage()); @@ -489,6 +858,76 @@ public void createSegment(String segmentName, String tier1GatewayName, String ga } } + protected String getIpDiscoveryProfilePath(String profileId) { + if (StringUtils.isBlank(profileId)) { + return null; + } + IpDiscoveryProfiles profiles = (IpDiscoveryProfiles) nsxService.apply(IpDiscoveryProfiles.class); + IPDiscoveryProfile profile = profiles.get(profileId); + return validateProfile(profileId, profile.getId(), profile.getPath(), "/infra/ip-discovery-profiles/", profile.getMarkedForDelete()); + } + + protected String getMacDiscoveryProfilePath(String profileId) { + if (StringUtils.isBlank(profileId)) { + return null; + } + MacDiscoveryProfiles profiles = (MacDiscoveryProfiles) nsxService.apply(MacDiscoveryProfiles.class); + MacDiscoveryProfile profile = profiles.get(profileId); + return validateProfile(profileId, profile.getId(), profile.getPath(), "/infra/mac-discovery-profiles/", profile.getMarkedForDelete()); + } + + protected String getSegmentSecurityProfilePath(String profileId) { + if (StringUtils.isBlank(profileId)) { + return null; + } + SegmentSecurityProfiles profiles = (SegmentSecurityProfiles) nsxService.apply(SegmentSecurityProfiles.class); + SegmentSecurityProfile profile = profiles.get(profileId); + return validateProfile(profileId, profile.getId(), profile.getPath(), "/infra/segment-security-profiles/", profile.getMarkedForDelete()); + } + + protected String validateProfile(String requestedId, String resolvedId, String profilePath, + String expectedPathPrefix, Boolean markedForDelete) { + if (!Objects.equals(requestedId, resolvedId)) { + throw new CloudRuntimeException(String.format("NSX returned profile %s while resolving requested profile %s", resolvedId, requestedId)); + } + if (!Objects.equals(expectedPathPrefix + requestedId, profilePath)) { + throw new CloudRuntimeException(String.format("NSX profile %s did not return a canonical resource path of the expected type", requestedId)); + } + if (Boolean.TRUE.equals(markedForDelete)) { + throw new CloudRuntimeException(String.format("NSX profile %s is marked for deletion", requestedId)); + } + return profilePath; + } + + protected List getSegmentProfileBindings(String ipDiscoveryProfilePath, String macDiscoveryProfilePath, + String segmentSecurityProfilePath) { + List bindings = new ArrayList<>(); + if (StringUtils.isNotBlank(ipDiscoveryProfilePath) || StringUtils.isNotBlank(macDiscoveryProfilePath)) { + SegmentDiscoveryProfileBindingMap binding = new SegmentDiscoveryProfileBindingMap.Builder() + .setResourceType(SEGMENT_DISCOVERY_PROFILE_BINDING_RESOURCE_TYPE) + .setId(SEGMENT_DISCOVERY_PROFILE_BINDING_ID) + .setIpDiscoveryProfilePath(ipDiscoveryProfilePath) + .setMacDiscoveryProfilePath(macDiscoveryProfilePath) + .build(); + bindings.add(new ChildSegmentDiscoveryProfileBindingMap.Builder() + .setId(SEGMENT_DISCOVERY_PROFILE_BINDING_ID) + .setSegmentDiscoveryProfileBindingMap(binding) + .build()); + } + if (StringUtils.isNotBlank(segmentSecurityProfilePath)) { + SegmentSecurityProfileBindingMap binding = new SegmentSecurityProfileBindingMap.Builder() + .setResourceType(SEGMENT_SECURITY_PROFILE_BINDING_RESOURCE_TYPE) + .setId(SEGMENT_SECURITY_PROFILE_BINDING_ID) + .setSegmentSecurityProfilePath(segmentSecurityProfilePath) + .build(); + bindings.add(new ChildSegmentSecurityProfileBindingMap.Builder() + .setId(SEGMENT_SECURITY_PROFILE_BINDING_ID) + .setSegmentSecurityProfileBindingMap(binding) + .build()); + } + return bindings; + } + public void deleteSegment(long zoneId, long domainId, long accountId, Long vpcId, long networkId, String segmentName) { try { removeSegmentDistributedFirewallRules(segmentName); @@ -1236,4 +1675,879 @@ private String getGroupPath(String segmentName) { return matchingGroup.map(Group::getPath).orElse(null); } + + public void createVpnService(String tier1GatewayName, String localEndpointIp) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String localEndpointName = getVpnLocalEndpointName(vpnServiceName); + try { + ensureTier1IpsecLocalEndpointAdvertisement(tier1GatewayName); + IpsecVpnServices vpnServices = (IpsecVpnServices) nsxService.apply(IpsecVpnServices.class); + IPSecVpnService vpnService = new IPSecVpnService.Builder() + .setId(vpnServiceName) + .setDisplayName(vpnServiceName) + .setEnabled(true) + .build(); + vpnServices.patch(tier1GatewayName, vpnServiceName, vpnService); + + LocalEndpoints localEndpoints = (LocalEndpoints) nsxService.apply(LocalEndpoints.class); + IPSecVpnLocalEndpoint localEndpoint = new IPSecVpnLocalEndpoint.Builder() + .setId(localEndpointName) + .setDisplayName(localEndpointName) + .setLocalAddress(localEndpointIp) + .setLocalId(localEndpointIp) + .build(); + localEndpoints.patch(tier1GatewayName, vpnServiceName, localEndpointName, localEndpoint); + + ensureVpnNatExemptions(tier1GatewayName, localEndpointIp); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create NSX IPSec VPN service %s on tier-1 gateway %s, due to: %s", + vpnServiceName, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + /** + * Returns whether the CloudStack-owned VPN service already exists. The resource uses this + * preflight result to avoid deleting a valid service when an idempotent create request fails + * after an ambiguous timeout. + */ + public boolean isVpnServicePresent(String tier1GatewayName) { + try { + IpsecVpnServices vpnServices = (IpsecVpnServices) nsxService.apply(IpsecVpnServices.class); + return vpnServices.get(tier1GatewayName, getVpnServiceName(tier1GatewayName)) != null; + } catch (NotFound e) { + return false; + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format("Failed to check NSX IPSec VPN service on tier-1 gateway %s, due to: %s", + tier1GatewayName, ae.getErrorMessage()), error); + } + } + + /** + * The local endpoint IP realizes as a Tier-1 loopback and is routable only when the Tier-1 + * advertises TIER1_IPSEC_LOCAL_ENDPOINT; gateways created by this plugin always do, but + * gateways created before that behavior are patched here + */ + private void ensureTier1IpsecLocalEndpointAdvertisement(String tier1GatewayName) { + Tier1 tier1 = getTier1Gateway(tier1GatewayName); + if (tier1 == null) { + throw new CloudRuntimeException(String.format("The Tier 1 Gateway %s does not exist", tier1GatewayName)); + } + List advertisementTypes = tier1.getRouteAdvertisementTypes(); + if (CollectionUtils.isNotEmpty(advertisementTypes) + && advertisementTypes.contains(RouteAdvertisementType.TIER1_IPSEC_LOCAL_ENDPOINT.name())) { + return; + } + List updatedTypes = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(advertisementTypes)) { + updatedTypes.addAll(advertisementTypes); + } + updatedTypes.add(RouteAdvertisementType.TIER1_IPSEC_LOCAL_ENDPOINT.name()); + Tier1s tier1service = (Tier1s) nsxService.apply(Tier1s.class); + Tier1 tier1Update = new Tier1.Builder() + .setRouteAdvertisementTypes(updatedTypes) + .build(); + tier1service.patch(tier1GatewayName, tier1Update); + } + + public void deleteVpnService(String tier1GatewayName) { + if (getTier1Gateway(tier1GatewayName) == null) { + // On VPC teardown the tier-1 gateway is removed (along with its VPN objects) before the + // VPN gateway cleanup runs + logger.debug("The Tier 1 Gateway {} does not exist, skipping the removal of its VPN service", tier1GatewayName); + return; + } + removeTier1VpnResources(tier1GatewayName); + restoreSourceNatRuleSequence(tier1GatewayName); + } + + public VpnSessionProvisioningResult createRouteBasedVpnSession(String tier1GatewayName, String connectionUuid, + String peerAddress, String psk, String ikePolicy, + String espPolicy, Long ikeLifetime, Long espLifetime, + boolean dpdEnabled, String ikeVersion, boolean passive, + String vtiLocalIp, int vtiPrefixLength) { + return retryWhileMarkedForDeletion(connectionUuid, () -> doCreateRouteBasedVpnSession(tier1GatewayName, + connectionUuid, peerAddress, psk, ikePolicy, espPolicy, ikeLifetime, espLifetime, dpdEnabled, + ikeVersion, passive, vtiLocalIp, vtiPrefixLength)); + } + + /** + * NSX reaps deleted policy objects asynchronously and rejects a create under a path that is still + * marked for deletion. All VPN objects use deterministic IDs and PATCH/upsert semantics, so a + * retry of the complete idempotent operation is safe and preserves already-existing objects. + */ + private T retryWhileMarkedForDeletion(String connectionUuid, Supplier operation) { + CloudRuntimeException lastFailure = null; + for (int attempt = 1; attempt <= VPN_MARKED_FOR_DELETION_RETRIES; attempt++) { + try { + return operation.get(); + } catch (CloudRuntimeException e) { + lastFailure = e; + if (attempt == VPN_MARKED_FOR_DELETION_RETRIES || !isMarkedForDeletionError(e)) { + throw e; + } + logger.info("A VPN object for connection {} is still being purged by NSX, retrying in {}s (attempt {}/{})", + connectionUuid, VPN_MARKED_FOR_DELETION_RETRY_INTERVAL_SECS, attempt, VPN_MARKED_FOR_DELETION_RETRIES); + try { + Thread.sleep(VPN_MARKED_FOR_DELETION_RETRY_INTERVAL_SECS * 1000L); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + throw lastFailure; + } + + private boolean isMarkedForDeletionError(CloudRuntimeException e) { + return e.getMessage() != null && e.getMessage().contains("marked for deletion"); + } + + private VpnSessionProvisioningResult doCreateRouteBasedVpnSession(String tier1GatewayName, String connectionUuid, + String peerAddress, String psk, String ikePolicy, + String espPolicy, Long ikeLifetime, Long espLifetime, + boolean dpdEnabled, String ikeVersion, boolean passive, + String vtiLocalIp, int vtiPrefixLength) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String localEndpointName = getVpnLocalEndpointName(vpnServiceName); + String sessionName = getVpnSessionName(connectionUuid); + String ikeProfileName = getVpnIkeProfileName(connectionUuid); + String espProfileName = getVpnEspProfileName(connectionUuid); + String dpdProfileName = getVpnDpdProfileName(connectionUuid); + try { + IpsecVpnIkeProfiles ikeProfiles = (IpsecVpnIkeProfiles) nsxService.apply(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles espProfiles = (IpsecVpnTunnelProfiles) nsxService.apply(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = (IpsecVpnDpdProfiles) nsxService.apply(IpsecVpnDpdProfiles.class); + Sessions sessions = (Sessions) nsxService.apply(Sessions.class); + boolean sessionExisted = isVpnSessionPresent(sessions, tier1GatewayName, vpnServiceName, sessionName); + if (sessionExisted) { + // Disable the existing session before replacing any referenced profiles so a failed + // reconciliation cannot leave an active tunnel using a partially updated policy. + updateVpnConnectionState(tier1GatewayName, connectionUuid, false); + } + boolean ikeProfileExisted = isVpnIkeProfilePresent(ikeProfiles, ikeProfileName); + boolean espProfileExisted = isVpnTunnelProfilePresent(espProfiles, espProfileName); + boolean dpdProfileExisted = isVpnDpdProfilePresent(dpdProfiles, dpdProfileName); + + try { + IPSecVpnIkeProfile ikeProfile = new IPSecVpnIkeProfile.Builder() + .setId(ikeProfileName) + .setDisplayName(ikeProfileName) + .setEncryptionAlgorithms(NsxVpnCryptoUtils.getEncryptionAlgorithms(ikePolicy)) + .setDigestAlgorithms(NsxVpnCryptoUtils.getDigestAlgorithms(ikePolicy)) + .setDhGroups(NsxVpnCryptoUtils.getDhGroups(ikePolicy)) + .setIkeVersion(NsxVpnCryptoUtils.getIkeVersion(ikeVersion)) + .setSaLifeTime(ikeLifetime) + .build(); + ikeProfiles.patch(ikeProfileName, ikeProfile); + + List espDhGroups = NsxVpnCryptoUtils.getDhGroups(espPolicy); + IPSecVpnTunnelProfile.Builder espProfileBuilder = new IPSecVpnTunnelProfile.Builder() + .setId(espProfileName) + .setDisplayName(espProfileName) + .setEncryptionAlgorithms(NsxVpnCryptoUtils.getEncryptionAlgorithms(espPolicy)) + .setDigestAlgorithms(NsxVpnCryptoUtils.getDigestAlgorithms(espPolicy)) + .setEnablePerfectForwardSecrecy(!espDhGroups.isEmpty()) + .setSaLifeTime(espLifetime); + if (!espDhGroups.isEmpty()) { + espProfileBuilder.setDhGroups(espDhGroups); + } + espProfiles.patch(espProfileName, espProfileBuilder.build()); + + // On demand probing only checks the peer when there is traffic to send and nothing has been + // heard back, so an idle tunnel is not torn down for want of a probe response the way the + // periodic default does; CloudStack only exposes DPD as a flag, hence the fixed timers + IPSecVpnDpdProfile dpdProfile = new IPSecVpnDpdProfile.Builder() + .setId(dpdProfileName) + .setDisplayName(dpdProfileName) + .setEnabled(dpdEnabled) + .setDpdProbeMode(IPSecVpnDpdProfile.DPD_PROBE_MODE_ON_DEMAND) + .setDpdProbeInterval(VPN_DPD_PROBE_INTERVAL_SECS) + .setRetryCount(VPN_DPD_RETRY_COUNT) + .build(); + dpdProfiles.patch(dpdProfileName, dpdProfile); + + IPSecVpnTunnelInterface tunnelInterface = new IPSecVpnTunnelInterface.Builder() + .setId(VPN_DEFAULT_TUNNEL_INTERFACE_NAME) + .setDisplayName(VPN_DEFAULT_TUNNEL_INTERFACE_NAME) + .setIpSubnets(List.of(new TunnelInterfaceIPSubnet.Builder() + .setIpAddresses(List.of(vtiLocalIp)) + .setPrefixLength((long) vtiPrefixLength) + .build())) + .build(); + RouteBasedIPSecVpnSession session = new RouteBasedIPSecVpnSession.Builder() + .setId(sessionName) + .setDisplayName(sessionName) + .setEnabled(false) + .setAuthenticationMode(IPSecVpnSession.AUTHENTICATION_MODE_PSK) + .setPsk(psk) + .setPeerAddress(peerAddress) + .setPeerId(peerAddress) + .setConnectionInitiationMode(passive ? IPSecVpnSession.CONNECTION_INITIATION_MODE_RESPOND_ONLY + : IPSecVpnSession.CONNECTION_INITIATION_MODE_INITIATOR) + .setIkeProfilePath(IPSEC_VPN_IKE_PROFILES_PATH_PREFIX + ikeProfileName) + .setTunnelProfilePath(IPSEC_VPN_TUNNEL_PROFILES_PATH_PREFIX + espProfileName) + .setDpdProfilePath(IPSEC_VPN_DPD_PROFILES_PATH_PREFIX + dpdProfileName) + .setLocalEndpointPath(getVpnLocalEndpointPath(tier1GatewayName, vpnServiceName, localEndpointName)) + .setTunnelInterfaces(List.of(tunnelInterface)) + .build(); + sessions.patch(tier1GatewayName, vpnServiceName, sessionName, session); + return sessionExisted ? VpnSessionProvisioningResult.PREEXISTING + : VpnSessionProvisioningResult.CREATED; + } catch (RuntimeException e) { + if (!sessionExisted) { + boolean sessionRemoved = deleteVpnSessionAfterCreateFailure(sessions, tier1GatewayName, + vpnServiceName, sessionName); + if (sessionRemoved) { + deleteNewVpnProfilesAfterCreateFailure(ikeProfiles, espProfiles, dpdProfiles, + connectionUuid, ikeProfileExisted, espProfileExisted, dpdProfileExisted); + } + } + throw e; + } + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to create NSX IPSec VPN session %s on tier-1 gateway %s, due to: %s", + sessionName, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private boolean isVpnSessionPresent(Sessions sessions, String tier1GatewayName, String vpnServiceName, + String sessionName) { + try { + return sessions.get(tier1GatewayName, vpnServiceName, sessionName) != null; + } catch (NotFound e) { + return false; + } + } + + private boolean isVpnIkeProfilePresent(IpsecVpnIkeProfiles profiles, String profileName) { + try { + return profiles.get(profileName) != null; + } catch (NotFound e) { + return false; + } + } + + private boolean isVpnTunnelProfilePresent(IpsecVpnTunnelProfiles profiles, String profileName) { + try { + return profiles.get(profileName) != null; + } catch (NotFound e) { + return false; + } + } + + private boolean isVpnDpdProfilePresent(IpsecVpnDpdProfiles profiles, String profileName) { + try { + return profiles.get(profileName) != null; + } catch (NotFound e) { + return false; + } + } + + private boolean deleteVpnSessionAfterCreateFailure(Sessions sessions, String tier1GatewayName, + String vpnServiceName, String sessionName) { + try { + sessions.delete(tier1GatewayName, vpnServiceName, sessionName); + return true; + } catch (NotFound e) { + logger.debug("The partially created VPN session {} on tier-1 gateway {} was not present during cleanup", + sessionName, tier1GatewayName); + return true; + } catch (Error e) { + logger.warn("Failed to remove the partially created VPN session {} on tier-1 gateway {} after creation failed: {}", + sessionName, tier1GatewayName, e.getMessage()); + return false; + } catch (RuntimeException e) { + logger.warn("Failed to remove the partially created VPN session {} on tier-1 gateway {} after creation failed: {}", + sessionName, tier1GatewayName, e.getMessage()); + return false; + } + } + + private void deleteNewVpnProfilesAfterCreateFailure(IpsecVpnIkeProfiles ikeProfiles, + IpsecVpnTunnelProfiles espProfiles, + IpsecVpnDpdProfiles dpdProfiles, + String connectionUuid, + boolean ikeProfileExisted, + boolean espProfileExisted, + boolean dpdProfileExisted) { + if (!ikeProfileExisted) { + deleteVpnProfileAfterCreateFailure(() -> ikeProfiles.delete(getVpnIkeProfileName(connectionUuid)), + "IKE", connectionUuid); + } + if (!espProfileExisted) { + deleteVpnProfileAfterCreateFailure(() -> espProfiles.delete(getVpnEspProfileName(connectionUuid)), + "tunnel", connectionUuid); + } + if (!dpdProfileExisted) { + deleteVpnProfileAfterCreateFailure(() -> dpdProfiles.delete(getVpnDpdProfileName(connectionUuid)), + "DPD", connectionUuid); + } + } + + private void deleteVpnProfileAfterCreateFailure(Runnable deleteAction, String profileType, + String connectionUuid) { + try { + deleteAction.run(); + } catch (NotFound e) { + logger.debug("The partially created {} profile of VPN connection {} was absent during cleanup", + profileType, connectionUuid); + } catch (RuntimeException e) { + logger.warn("Failed to remove the partially created {} profile of VPN connection {}: {}", + profileType, connectionUuid, e.getMessage()); + } + } + + public void addVpnConnectionRoutes(String tier1GatewayName, String connectionUuid, List peerCidrs, + String vtiPeerIp, String vpcCidr) { + retryWhileMarkedForDeletion(connectionUuid, () -> { + doAddVpnConnectionRoutes(tier1GatewayName, connectionUuid, peerCidrs, vtiPeerIp, vpcCidr); + return null; + }); + } + + private void doAddVpnConnectionRoutes(String tier1GatewayName, String connectionUuid, List peerCidrs, + String vtiPeerIp, String vpcCidr) { + try { + com.vmware.nsx_policy.infra.tier_1s.StaticRoutes staticRoutesService = + (com.vmware.nsx_policy.infra.tier_1s.StaticRoutes) nsxService.apply(com.vmware.nsx_policy.infra.tier_1s.StaticRoutes.class); + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + Set desiredRouteIds = new HashSet<>(); + Set desiredNoSnatRuleIds = new HashSet<>(); + for (int i = 0; i < peerCidrs.size(); i++) { + String peerCidr = peerCidrs.get(i); + String routeName = getVpnStaticRouteName(connectionUuid, i); + desiredRouteIds.add(routeName); + com.vmware.nsx_policy.model.StaticRoutes staticRoute = new com.vmware.nsx_policy.model.StaticRoutes.Builder() + .setId(routeName) + .setDisplayName(routeName) + .setNetwork(peerCidr) + .setNextHops(List.of(new RouterNexthop.Builder().setIpAddress(vtiPeerIp).build())) + .build(); + staticRoutesService.patch(tier1GatewayName, routeName, staticRoute); + + // Route-based VPN does not bypass NAT: without a NO_SNAT rule the tier-1 match-any + // SNAT would rewrite VPC-to-remote traffic before it enters the tunnel + String noSnatRuleName = getVpnNoSnatRuleName(connectionUuid, i); + desiredNoSnatRuleIds.add(noSnatRuleName); + PolicyNatRule noSnatRule = new PolicyNatRule.Builder() + .setId(noSnatRuleName) + .setDisplayName(noSnatRuleName) + .setAction(NatAction.NO_SNAT.name()) + .setSourceNetwork(vpcCidr) + .setDestinationNetwork(peerCidr) + .setSequenceNumber(VPN_NO_SNAT_SEQUENCE_NUMBER) + .setEnabled(true) + .build(); + natService.patch(tier1GatewayName, NatId.USER.name(), noSnatRuleName, noSnatRule); + } + // Keep existing routes and exemptions in place until every desired object has been + // accepted. This makes an idempotent retry non-destructive if an NSX PATCH fails. + deleteVpnStaticRoutesByPrefix(tier1GatewayName, getVpnStaticRouteNamePrefix(connectionUuid), + desiredRouteIds); + deleteVpnNoSnatRulesByPrefix(tier1GatewayName, getVpnNoSnatRuleNamePrefix(connectionUuid), + desiredNoSnatRuleIds); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to add the routes for NSX IPSec VPN connection %s on tier-1 gateway %s, due to: %s", + connectionUuid, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void deleteVpnConnection(String tier1GatewayName, String connectionUuid) { + RuntimeException failure = null; + // Delete by prefix instead of recomputing names from the current peer CIDR list: the + // customer gateway's CIDRs may have changed since the routes and NO_SNAT rules were created + failure = runVpnCleanupStep(failure, "static routes", connectionUuid, + () -> deleteVpnStaticRoutesByPrefix(tier1GatewayName, getVpnStaticRouteNamePrefix(connectionUuid))); + failure = runVpnCleanupStep(failure, "NO_SNAT rules", connectionUuid, + () -> deleteVpnNoSnatRulesByPrefix(tier1GatewayName, getVpnNoSnatRuleNamePrefix(connectionUuid))); + failure = runVpnCleanupStep(failure, "session", connectionUuid, + () -> deleteVpnSession(tier1GatewayName, connectionUuid)); + failure = runVpnCleanupStep(failure, "profiles", connectionUuid, + () -> deleteVpnSessionProfiles(connectionUuid)); + throwVpnCleanupFailure(failure, connectionUuid); + } + + private void deleteVpnSession(String tier1GatewayName, String connectionUuid) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String sessionName = getVpnSessionName(connectionUuid); + try { + Sessions sessions = (Sessions) nsxService.apply(Sessions.class); + sessions.delete(tier1GatewayName, vpnServiceName, sessionName); + } catch (NotFound e) { + logger.debug("The VPN session {} on tier-1 gateway {} no longer exists, skipping deletion", + sessionName, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the NSX IPSec VPN session %s on tier-1 gateway %s, due to: %s", + sessionName, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + public void updateVpnConnectionState(String tier1GatewayName, String connectionUuid, boolean enabled) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String sessionName = getVpnSessionName(connectionUuid); + try { + Sessions sessions = (Sessions) nsxService.apply(Sessions.class); + Structure current = sessions.showsensitivedata(tier1GatewayName, vpnServiceName, sessionName); + if (current == null) { + throw new CloudRuntimeException(String.format( + "NSX returned no data for IPSec VPN session %s on tier-1 gateway %s", + sessionName, tier1GatewayName)); + } + if (!current._hasTypeNameOf(RouteBasedIPSecVpnSession.class)) { + throw new CloudRuntimeException(String.format( + "IPSec VPN session %s on tier-1 gateway %s is not route-based", + sessionName, tier1GatewayName)); + } + RouteBasedIPSecVpnSession update = current._convertTo(RouteBasedIPSecVpnSession.class); + if (update.getRevision() == null) { + throw new CloudRuntimeException(String.format( + "NSX returned no revision for IPSec VPN session %s on tier-1 gateway %s", + sessionName, tier1GatewayName)); + } + if (IPSecVpnSession.AUTHENTICATION_MODE_PSK.equals(update.getAuthenticationMode()) + && update.getPsk() == null) { + throw new CloudRuntimeException(String.format( + "NSX did not return sensitive authentication data for IPSec VPN session %s on tier-1 gateway %s", + sessionName, tier1GatewayName)); + } + update.setEnabled(enabled); + sessions.update(tier1GatewayName, vpnServiceName, sessionName, update); + } catch (NotFound e) { + if (enabled) { + throw new CloudRuntimeException(String.format( + "Cannot enable NSX IPSec VPN session %s on tier-1 gateway %s because it does not exist", + sessionName, tier1GatewayName), e); + } + logger.debug("The VPN session {} no longer exists on tier-1 gateway {}, skipping state update", + sessionName, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to update the state of NSX IPSec VPN session %s on tier-1 gateway %s, due to: %s", + sessionName, tier1GatewayName, ae.getErrorMessage()), error); + } + if (!enabled) { + deleteVpnStaticRoutesByPrefix(tier1GatewayName, getVpnStaticRouteNamePrefix(connectionUuid)); + deleteVpnNoSnatRulesByPrefix(tier1GatewayName, getVpnNoSnatRuleNamePrefix(connectionUuid)); + } + } + + /** + * Removes every object created for a connection when route or NAT programming fails after the + * session itself was created. This is also used by the permanent connection-delete path. + */ + public void rollbackVpnConnection(String tier1GatewayName, String connectionUuid) { + deleteVpnConnection(tier1GatewayName, connectionUuid); + } + + private void deleteVpnStaticRoutesByPrefix(String tier1GatewayName, String routeNamePrefix) { + deleteVpnStaticRoutesByPrefix(tier1GatewayName, routeNamePrefix, Set.of()); + } + + private void deleteVpnStaticRoutesByPrefix(String tier1GatewayName, String routeNamePrefix, + Set retainedRouteIds) { + com.vmware.nsx_policy.infra.tier_1s.StaticRoutes staticRoutesService = + (com.vmware.nsx_policy.infra.tier_1s.StaticRoutes) nsxService.apply(com.vmware.nsx_policy.infra.tier_1s.StaticRoutes.class); + try { + List staticRoutes = + PagedFetcher.withPageFetcher( + cursor -> staticRoutesService.list(tier1GatewayName, cursor, false, null, null, null, null) + ).cursorExtractor(StaticRoutesListResult::getCursor) + .itemsExtractor(StaticRoutesListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll() + .getResults(); + if (CollectionUtils.isEmpty(staticRoutes)) { + return; + } + RuntimeException failure = null; + for (com.vmware.nsx_policy.model.StaticRoutes staticRoute : staticRoutes) { + if (staticRoute.getId() != null && staticRoute.getId().startsWith(routeNamePrefix) + && !retainedRouteIds.contains(staticRoute.getId())) { + logger.debug("Removing the VPN static route {} from tier-1 gateway {}", staticRoute.getId(), tier1GatewayName); + String routeId = staticRoute.getId(); + failure = runVpnCleanupStep(failure, String.format("static route %s", routeId), routeNamePrefix, + () -> deleteVpnStaticRoute(staticRoutesService, tier1GatewayName, routeId)); + } + } + throwVpnPrefixCleanupFailure(failure, "static routes", routeNamePrefix, tier1GatewayName); + } catch (NotFound e) { + logger.debug("No static routes matching the prefix {} are left on tier-1 gateway {}, skipping deletion", + routeNamePrefix, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the VPN static routes matching the prefix %s on tier-1 gateway %s, due to: %s", + routeNamePrefix, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private void deleteVpnStaticRoute(com.vmware.nsx_policy.infra.tier_1s.StaticRoutes staticRoutesService, + String tier1GatewayName, String routeId) { + try { + staticRoutesService.delete(tier1GatewayName, routeId); + } catch (NotFound e) { + logger.debug("The VPN static route {} on tier-1 gateway {} no longer exists, skipping deletion", + routeId, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to delete the VPN static route %s on tier-1 gateway %s, due to: %s", + routeId, tier1GatewayName, ae.getErrorMessage()), error); + } + } + + private void deleteVpnNoSnatRulesByPrefix(String tier1GatewayName, String ruleNamePrefix) { + deleteVpnNoSnatRulesByPrefix(tier1GatewayName, ruleNamePrefix, Set.of()); + } + + private void deleteVpnNoSnatRulesByPrefix(String tier1GatewayName, String ruleNamePrefix, + Set retainedRuleIds) { + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + try { + List natRules = PagedFetcher.withPageFetcher( + cursor -> natService.list(tier1GatewayName, NatId.USER.name(), cursor, false, null, null, null, null) + ).cursorExtractor(PolicyNatRuleListResult::getCursor) + .itemsExtractor(PolicyNatRuleListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll() + .getResults(); + if (CollectionUtils.isEmpty(natRules)) { + return; + } + RuntimeException failure = null; + for (PolicyNatRule natRule : natRules) { + if (natRule.getId() != null && natRule.getId().startsWith(ruleNamePrefix) + && !retainedRuleIds.contains(natRule.getId())) { + logger.debug("Removing the VPN NO_SNAT rule {} from tier-1 gateway {}", natRule.getId(), tier1GatewayName); + String ruleId = natRule.getId(); + failure = runVpnCleanupStep(failure, String.format("NO_SNAT rule %s", ruleId), ruleNamePrefix, + () -> deleteVpnNoSnatRule(natService, tier1GatewayName, ruleId)); + } + } + throwVpnPrefixCleanupFailure(failure, "NO_SNAT rules", ruleNamePrefix, tier1GatewayName); + } catch (NotFound e) { + logger.debug("No NO_SNAT rules matching the prefix {} are left on tier-1 gateway {}, skipping deletion", + ruleNamePrefix, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the VPN NO_SNAT rules matching the prefix %s on tier-1 gateway %s, due to: %s", + ruleNamePrefix, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private void deleteVpnNoSnatRule(NatRules natService, String tier1GatewayName, String ruleId) { + try { + natService.delete(tier1GatewayName, NatId.USER.name(), ruleId); + } catch (NotFound e) { + logger.debug("The VPN NO_SNAT rule {} on tier-1 gateway {} no longer exists, skipping deletion", + ruleId, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to delete the VPN NO_SNAT rule %s on tier-1 gateway %s, due to: %s", + ruleId, tier1GatewayName, ae.getErrorMessage()), error); + } + } + + private void throwVpnPrefixCleanupFailure(RuntimeException failure, String resource, String prefix, + String tier1GatewayName) { + if (failure != null) { + throw new CloudRuntimeException(String.format( + "Failed to remove all NSX VPN %s matching prefix %s on tier-1 gateway %s", + resource, prefix, tier1GatewayName), failure); + } + } + + private void deleteVpnSessionProfiles(String connectionUuid) { + RuntimeException failure = null; + failure = runVpnCleanupStep(failure, "IKE profile", connectionUuid, + () -> deleteVpnIkeProfile(connectionUuid)); + failure = runVpnCleanupStep(failure, "tunnel profile", connectionUuid, + () -> deleteVpnTunnelProfile(connectionUuid)); + failure = runVpnCleanupStep(failure, "DPD profile", connectionUuid, + () -> deleteVpnDpdProfile(connectionUuid)); + throwVpnCleanupFailure(failure, connectionUuid); + } + + private void deleteVpnIkeProfile(String connectionUuid) { + try { + IpsecVpnIkeProfiles ikeProfiles = (IpsecVpnIkeProfiles) nsxService.apply(IpsecVpnIkeProfiles.class); + ikeProfiles.delete(getVpnIkeProfileName(connectionUuid)); + } catch (NotFound e) { + logger.debug("The IKE profile of VPN connection {} no longer exists, skipping deletion", connectionUuid); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the IKE profile of VPN connection %s, due to: %s", + connectionUuid, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private void deleteVpnTunnelProfile(String connectionUuid) { + try { + IpsecVpnTunnelProfiles espProfiles = (IpsecVpnTunnelProfiles) nsxService.apply(IpsecVpnTunnelProfiles.class); + espProfiles.delete(getVpnEspProfileName(connectionUuid)); + } catch (NotFound e) { + logger.debug("The tunnel profile of VPN connection {} no longer exists, skipping deletion", connectionUuid); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the tunnel profile of VPN connection %s, due to: %s", + connectionUuid, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private void deleteVpnDpdProfile(String connectionUuid) { + try { + IpsecVpnDpdProfiles dpdProfiles = (IpsecVpnDpdProfiles) nsxService.apply(IpsecVpnDpdProfiles.class); + dpdProfiles.delete(getVpnDpdProfileName(connectionUuid)); + } catch (NotFound e) { + logger.debug("The DPD profile of VPN connection {} no longer exists, skipping deletion", connectionUuid); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to delete the DPD profile of VPN connection %s, due to: %s", + connectionUuid, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private RuntimeException runVpnCleanupStep(RuntimeException failure, String resource, String connectionUuid, + Runnable cleanup) { + try { + cleanup.run(); + } catch (RuntimeException e) { + if (failure == null) { + return e; + } + failure.addSuppressed(e); + logger.warn("Failed to remove NSX VPN {} for connection {} after an earlier cleanup failure: {}", + resource, connectionUuid, e.getMessage()); + } + return failure; + } + + private void throwVpnCleanupFailure(RuntimeException failure, String connectionUuid) { + if (failure == null) { + return; + } + if (failure instanceof CloudRuntimeException) { + throw (CloudRuntimeException) failure; + } + throw new CloudRuntimeException(String.format( + "Failed to remove all NSX VPN resources for connection %s: %s", connectionUuid, failure.getMessage()), failure); + } + + public String getVpnSessionStatus(String tier1GatewayName, String connectionUuid) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String sessionName = getVpnSessionName(connectionUuid); + try { + DetailedStatus detailedStatusService = (DetailedStatus) nsxService.apply(DetailedStatus.class); + AggregateIPSecVpnSessionStatus aggregateStatus = detailedStatusService.get(tier1GatewayName, vpnServiceName, sessionName, null, null); + List results = aggregateStatus == null ? null : aggregateStatus.getResults(); + if (CollectionUtils.isEmpty(results)) { + return VPN_SESSION_STATUS_UNKNOWN; + } + List statuses = results.stream() + .map(result -> result._convertTo(IPSecVpnSessionStatusNsxt.class).getRuntimeStatus()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (statuses.isEmpty()) { + return VPN_SESSION_STATUS_UNKNOWN; + } + if (statuses.contains(IPSecVpnSessionStatusNsxt.RUNTIME_STATUS_DOWN)) { + return IPSecVpnSessionStatusNsxt.RUNTIME_STATUS_DOWN; + } + if (statuses.stream().allMatch(IPSecVpnSessionStatusNsxt.RUNTIME_STATUS_UP::equals)) { + return IPSecVpnSessionStatusNsxt.RUNTIME_STATUS_UP; + } + return statuses.get(0); + } catch (NotFound e) { + logger.debug("The VPN session {} no longer exists on tier-1 gateway {}", sessionName, tier1GatewayName); + return VPN_SESSION_STATUS_NOT_FOUND; + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to get the status of NSX IPSec VPN session %s on tier-1 gateway %s, due to: %s", + sessionName, tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + /** + * VPN sessions, local endpoints, services and static routes must be removed before the tier-1 + * locale-services deletion during gateway teardown + */ + private void removeTier1VpnResources(String tier1Id) { + deleteVpnStaticRoutesByPrefix(tier1Id, getVpnSessionName("")); + deleteVpnNoSnatRulesByPrefix(tier1Id, getVpnSessionName("")); + deleteVpnLocalEndpointNoSnatRule(tier1Id); + try { + IpsecVpnServices vpnServices = (IpsecVpnServices) nsxService.apply(IpsecVpnServices.class); + List services = new ArrayList<>(PagedFetcher.withPageFetcher( + cursor -> vpnServices.list(tier1Id, cursor, false, null, null, false, null)) + .cursorExtractor(IPSecVpnServiceListResult::getCursor) + .itemsExtractor(IPSecVpnServiceListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll().getResults()); + // A Tier-1 may also carry VPN services owned by an operator or another integration. + // CloudStack owns exactly the deterministic service created for this gateway. + String cloudStackVpnServiceName = getVpnServiceName(tier1Id); + services.removeIf(service -> !cloudStackVpnServiceName.equals(service.getId())); + if (CollectionUtils.isEmpty(services)) { + return; + } + Sessions sessions = (Sessions) nsxService.apply(Sessions.class); + LocalEndpoints localEndpoints = (LocalEndpoints) nsxService.apply(LocalEndpoints.class); + for (IPSecVpnService service : services) { + List sessionResults = PagedFetcher.withPageFetcher( + cursor -> sessions.list(tier1Id, service.getId(), cursor, false, null, null, false, null)) + .cursorExtractor(IPSecVpnSessionListResult::getCursor) + .itemsExtractor(IPSecVpnSessionListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll().getResults(); + if (CollectionUtils.isNotEmpty(sessionResults)) { + String sessionNamePrefix = getVpnSessionName(""); + for (Structure result : sessionResults) { + IPSecVpnSession session = result._convertTo(IPSecVpnSession.class); + logger.debug("Removing VPN session {} from the VPN service {} of Tier 1 Gateway {}", session.getId(), service.getId(), tier1Id); + sessions.delete(tier1Id, service.getId(), session.getId()); + if (session.getId().startsWith(sessionNamePrefix)) { + deleteVpnSessionProfiles(session.getId().substring(sessionNamePrefix.length())); + } + } + } + List localEndpointResults = PagedFetcher.withPageFetcher( + cursor -> localEndpoints.list(tier1Id, service.getId(), cursor, false, null, null, false, null)) + .cursorExtractor(IPSecVpnLocalEndpointListResult::getCursor) + .itemsExtractor(IPSecVpnLocalEndpointListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll().getResults(); + if (CollectionUtils.isNotEmpty(localEndpointResults)) { + for (IPSecVpnLocalEndpoint localEndpoint : localEndpointResults) { + logger.debug("Removing VPN local endpoint {} from the VPN service {} of Tier 1 Gateway {}", localEndpoint.getId(), service.getId(), tier1Id); + localEndpoints.delete(tier1Id, service.getId(), localEndpoint.getId()); + } + } + logger.debug("Removing VPN service {} from Tier 1 Gateway {}", service.getId(), tier1Id); + vpnServices.delete(tier1Id, service.getId()); + } + } catch (NotFound e) { + logger.debug("The VPN resources of the Tier 1 Gateway {} no longer exist, skipping deletion", tier1Id); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to remove the VPN resources of the Tier 1 Gateway %s, due to: %s", + tier1Id, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private void deleteVpnLocalEndpointNoSnatRule(String tier1GatewayName) { + String ruleName = getVpnLocalEndpointNoSnatRuleName(getVpnServiceName(tier1GatewayName)); + try { + NatRules natService = (NatRules) nsxService.apply(NatRules.class); + natService.delete(tier1GatewayName, NatId.USER.name(), ruleName); + } catch (NotFound e) { + logger.debug("The VPN local-endpoint no-SNAT rule {} no longer exists on tier-1 gateway {}", + ruleName, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + throw new CloudRuntimeException(String.format( + "Failed to delete the VPN local-endpoint no-SNAT rule %s on tier-1 gateway %s, due to: %s", + ruleName, tier1GatewayName, ae.getErrorMessage()), error); + } + } + + /** + * Lists the local VTI addresses of the route-based VPN sessions on a tier-1 gateway, excluding + * the session of the given connection; used to fail closed on deterministic VTI collisions. + */ + public Set getRouteBasedVpnSessionLocalVtiIps(String tier1GatewayName, String excludedConnectionUuid) { + String vpnServiceName = getVpnServiceName(tier1GatewayName); + String excludedSessionName = getVpnSessionName(excludedConnectionUuid); + Set vtiIps = new HashSet<>(); + try { + Sessions sessions = (Sessions) nsxService.apply(Sessions.class); + List sessionResults = PagedFetcher.withPageFetcher( + cursor -> sessions.list(tier1GatewayName, vpnServiceName, cursor, false, null, null, false, null)) + .cursorExtractor(IPSecVpnSessionListResult::getCursor) + .itemsExtractor(IPSecVpnSessionListResult::getResults) + .itemsSetter((page, allItems) -> { + page.setResults(allItems); + page.setResultCount((long) allItems.size()); + }) + .fetchAll().getResults(); + for (Structure result : sessionResults) { + IPSecVpnSession session = result._convertTo(IPSecVpnSession.class); + if (excludedSessionName.equals(session.getId()) + || !RouteBasedIPSecVpnSession.class.getSimpleName().equals(session.getResourceType())) { + continue; + } + RouteBasedIPSecVpnSession routeBasedSession = result._convertTo(RouteBasedIPSecVpnSession.class); + if (CollectionUtils.isEmpty(routeBasedSession.getTunnelInterfaces())) { + continue; + } + for (IPSecVpnTunnelInterface tunnelInterface : routeBasedSession.getTunnelInterfaces()) { + if (CollectionUtils.isEmpty(tunnelInterface.getIpSubnets())) { + continue; + } + for (TunnelInterfaceIPSubnet ipSubnet : tunnelInterface.getIpSubnets()) { + if (CollectionUtils.isNotEmpty(ipSubnet.getIpAddresses())) { + vtiIps.addAll(ipSubnet.getIpAddresses()); + } + } + } + } + } catch (NotFound e) { + logger.debug("The VPN service {} does not exist yet on tier-1 gateway {}, no VTI addresses are in use", + vpnServiceName, tier1GatewayName); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to list the VPN sessions on tier-1 gateway %s, due to: %s", + tier1GatewayName, ae.getErrorMessage()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + return vtiIps; + } + + private String getVpnLocalEndpointPath(String tier1GatewayName, String vpnServiceName, String localEndpointName) { + return TIER_1_GATEWAY_PATH_PREFIX + tier1GatewayName + "/ipsec-vpn-services/" + vpnServiceName + + "/local-endpoints/" + localEndpointName; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index 0486de96dd1b..1f120c779ff6 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -40,22 +40,32 @@ import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.Networks; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PublicIpAddress; import com.cloud.network.SDNProviderNetworkRule; +import com.cloud.network.Site2SiteCustomerGateway; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; import com.cloud.network.VirtualRouterProvider; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.network.dao.Site2SiteCustomerGatewayDao; +import com.cloud.network.dao.Site2SiteCustomerGatewayVO; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.Site2SiteVpnGatewayVO; import com.cloud.network.dao.VirtualRouterProviderDao; import com.cloud.network.element.DhcpServiceProvider; import com.cloud.network.element.DnsServiceProvider; @@ -64,19 +74,24 @@ import com.cloud.network.element.LoadBalancingServiceProvider; import com.cloud.network.element.NetworkACLServiceProvider; import com.cloud.network.element.PortForwardingServiceProvider; +import com.cloud.network.element.Site2SiteVpnServiceProvider; import com.cloud.network.element.StaticNatServiceProvider; import com.cloud.network.element.VirtualRouterElement; import com.cloud.network.element.VirtualRouterProviderVO; import com.cloud.network.element.VpcProvider; import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.nsx.NsxVpnGatewayResult; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.LoadBalancerContainer; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.PrivateGateway; import com.cloud.network.vpc.StaticRouteProfile; import com.cloud.network.vpc.Vpc; +import com.cloud.network.vpc.VpcService; +import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; @@ -108,14 +123,20 @@ import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd; import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd; import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.cloudstack.resource.NsxLoadBalancerMember; import org.apache.cloudstack.resource.NsxNetworkRule; import com.cloud.network.SDNProviderOpObject; +import org.apache.cloudstack.utils.NsxHelper; +import org.apache.cloudstack.utils.NsxVpnCryptoUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.cloudstack.resourcedetail.FirewallRuleDetailVO; +import org.apache.cloudstack.resourcedetail.UserIpAddressDetailVO; import org.apache.cloudstack.resourcedetail.dao.FirewallRuleDetailsDao; +import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -135,8 +156,10 @@ @Component public class NsxElement extends AdapterBase implements DhcpServiceProvider, DnsServiceProvider, VpcProvider, StaticNatServiceProvider, IpDeployer, PortForwardingServiceProvider, NetworkACLServiceProvider, - LoadBalancingServiceProvider, FirewallServiceProvider, InternalLoadBalancerElementService, ResourceStateAdapter, Listener { + LoadBalancingServiceProvider, FirewallServiceProvider, Site2SiteVpnServiceProvider, + InternalLoadBalancerElementService, ResourceStateAdapter, Listener { + protected static final String NSX_VPN_GATEWAY_IP_DETAIL = "nsxVpnGatewayIp"; @Inject AccountManager accountMgr; @@ -167,11 +190,29 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns @Inject LoadBalancerVMMapDao lbVmMapDao; @Inject + LoadBalancerDao loadBalancerDao; + @Inject VirtualRouterProviderDao vrProviderDao; @Inject PhysicalNetworkServiceProviderDao pNtwkSvcProviderDao; @Inject FirewallRuleDetailsDao firewallRuleDetailsDao; + @Inject + IpAddressManager ipAddressManager; + @Inject + VpcService vpcService; + @Inject + VpcManager vpcManager; + @Inject + Site2SiteVpnGatewayDao vpnGatewayDao; + @Inject + Site2SiteCustomerGatewayDao customerGatewayDao; + @Inject + UserIpAddressDetailsDao userIpAddressDetailsDao; + @Inject + FirewallRulesDao firewallRulesDao; + @Inject + PortForwardingRulesDao portForwardingRulesDao; protected Logger logger = LogManager.getLogger(getClass()); @@ -214,6 +255,11 @@ private static Map> initCapabil sourceNatCapabilities.put(Network.Capability.RedundantRouter, "true"); sourceNatCapabilities.put(Network.Capability.SupportedSourceNatTypes, "peraccount"); capabilities.put(Network.Service.SourceNat, sourceNatCapabilities); + + Map vpnCapabilities = new HashMap<>(); + vpnCapabilities.put(Network.Capability.SupportedVpnProtocols, "ipsec"); + vpnCapabilities.put(Network.Capability.VpnTypes, "s2svpn"); + capabilities.put(Network.Service.Vpn, vpnCapabilities); return capabilities; } @Override @@ -535,6 +581,9 @@ public boolean applyStaticNats(Network config, List rules) String networkResourceName = Objects.nonNull(vpc) ? vpc.getName() : network.getName(); boolean isVpcResource = Objects.nonNull(vpc); if (!staticNat.isForRevoke()) { + validatePublicIpVlan(ipAddressVO, config.getDataCenterId(), config.getAccountId(), + config.getDomainId(), isVpcResource ? networkResourceId : null, + isVpcResource ? null : networkResourceId); return nsxService.createStaticNatRule(config.getDataCenterId(), config.getDomainId(), config.getAccountId(), networkResourceId, networkResourceName, isVpcResource, vm.getId(), ipAddressVO.getAddress().addr(), staticNat.getDestIpAddress()); @@ -581,6 +630,9 @@ protected synchronized boolean applyPFRulesInternal(Network network, List rules) thro networkRule.setBaseRule(baseNetRule); networkRule.setMemberList(lbMembers); if (Arrays.asList(FirewallRule.State.Add, FirewallRule.State.Active).contains(loadBalancingRule.getState())) { + if (LoadBalancerContainer.Scheme.Public == loadBalancingRule.getScheme()) { + validatePublicIpVlan(publicIp, nsxObject.getZoneId(), nsxObject.getAccountId(), + nsxObject.getDomainId(), nsxObject.isVpcResource() ? nsxObject.getNetworkResourceId() : null, + nsxObject.isVpcResource() ? null : nsxObject.getNetworkResourceId()); + } result &= nsxService.createLbRule(networkRule); } else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) { result &= nsxService.deleteLbRule(networkRule); @@ -706,6 +763,14 @@ public boolean applyLBRules(Network network, List rules) thro return result; } + private void validatePublicIpVlan(IPAddressVO publicIp, long zoneId, long accountId, long domainId, + Long vpcId, Long networkId) { + if (publicIp == null) { + throw new CloudRuntimeException("The public IP address could not be found"); + } + nsxService.validatePublicIpVlan(zoneId, accountId, domainId, vpcId, networkId, publicIp.getVlanId()); + } + @Override public boolean validateLBRule(Network network, LoadBalancingRule rule) { return true; @@ -941,4 +1006,294 @@ public List> getCommands() { public boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address) { return nsxService.updateVpcSourceNatIp(vpc, address); } + + protected boolean isVpnProvidedByNsx(Vpc vpc) { + if (Objects.isNull(vpc)) { + return false; + } + if (vpcManager != null) { + return vpcManager.isProviderSupportServiceInVpc(vpc.getId(), Network.Service.Vpn, Network.Provider.Nsx); + } + return Objects.nonNull(vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId( + Network.Service.Vpn.getName(), Network.Provider.Nsx.getName(), vpc.getVpcOfferingId())); + } + + protected boolean isVpnProvidedByNsx(Vpc vpc, Site2SiteVpnGateway gateway) { + return vpc != null && ownsVpnGateway(gateway); + } + + @Override + public IpAddress acquireVpnGatewayIp(Vpc vpc, IpAddress requestedIp) { + if (!isVpnProvidedByNsx(vpc)) { + return null; + } + IPAddressVO ip; + boolean autoAcquired = false; + if (Objects.nonNull(requestedIp)) { + ip = validateRequestedVpnGatewayIp(vpc, requestedIp); + } else { + ip = allocateVpnGatewayIp(vpc); + autoAcquired = true; + } + if (!autoAcquired) { + try { + // Mark ownership first so ambiguous NSX responses remain recoverable. + userIpAddressDetailsDao.addDetail(ip.getId(), NSX_VPN_GATEWAY_IP_DETAIL, "false", false); + } catch (Exception e) { + throw new CloudRuntimeException(String.format( + "Failed to record NSX VPN ownership for requested IP %s of VPC %s", + ip.getAddress(), vpc.getName()), e); + } + } + boolean endpointMayBeInUse = true; + try { + NsxVpnGatewayResult result = nsxService.createVpnGateway(vpc, ip.getAddress().addr()); + endpointMayBeInUse = result.isEndpointMayBeInUse(); + if (!result.isSuccessful()) { + throw new CloudRuntimeException(String.format("The NSX VPN gateway service for VPC %s was not created: the provider returned an unsuccessful answer", + vpc.getName())); + } + } catch (Exception e) { + if (autoAcquired && !endpointMayBeInUse) { + try { + releaseAutoAcquiredVpnGatewayIp(ip); + } catch (Exception cleanupException) { + logger.warn("Failed to release the auto-acquired VPN gateway IP {} of VPC {} after creation failed: {}", + ip.getAddress(), vpc.getName(), cleanupException.getMessage()); + } + } else if (autoAcquired) { + logger.warn("Retaining auto-acquired VPN gateway IP {} for VPC {} because the NSX endpoint may still be using it", + ip.getAddress(), vpc.getName()); + } else if (!endpointMayBeInUse) { + try { + userIpAddressDetailsDao.removeDetail(ip.getId(), NSX_VPN_GATEWAY_IP_DETAIL); + } catch (Exception cleanupException) { + logger.warn("Failed to remove the NSX VPN ownership marker from requested IP {} of VPC {} after gateway creation failed: {}", + ip.getAddress(), vpc.getName(), cleanupException.getMessage()); + } + } + throw new CloudRuntimeException(String.format("Failed to create the NSX VPN gateway for VPC %s: %s", + vpc.getName(), e.getMessage()), e); + } + return ip; + } + + private IPAddressVO validateRequestedVpnGatewayIp(Vpc vpc, IpAddress requestedIp) { + IPAddressVO ip = ipAddressDao.findById(requestedIp.getId()); + if (Objects.isNull(ip) || !Objects.equals(ip.getVpcId(), vpc.getId()) + || !ip.readyToUse() || ip.getRemoved() != null || ip.getAddress() == null) { + throw new InvalidParameterValueException(String.format( + "The requested IP id %s is not an allocated, active IP associated to the VPC %s", + requestedIp.getId(), vpc.getName())); + } + if (ip.isSourceNat() || ip.isForSystemVms()) { + throw new InvalidParameterValueException(String.format( + "The requested IP %s cannot be used as the VPN gateway IP as it is a source NAT or system IP", ip.getAddress().addr())); + } + if (ip.isOneToOneNat() || !firewallRulesDao.listByIpAndNotRevoked(ip.getId()).isEmpty() + || !portForwardingRulesDao.listByIpAndNotRevoked(ip.getId()).isEmpty() + || !loadBalancerDao.listByIpAddress(ip.getId()).isEmpty()) { + throw new InvalidParameterValueException(String.format( + "The requested IP %s cannot be used as the VPN gateway IP as it is already in use by static NAT or network rules", ip.getAddress().addr())); + } + nsxService.validatePublicIpVlan(vpc.getZoneId(), vpc.getAccountId(), vpc.getDomainId(), + vpc.getId(), null, ip.getVlanId()); + return ip; + } + + private IPAddressVO allocateVpnGatewayIp(Vpc vpc) { + Account owner = accountMgr.getAccount(vpc.getAccountId()); + DataCenterVO zone = dataCenterDao.findById(vpc.getZoneId()); + IpAddress allocatedIp = null; + try { + Long publicVlanId = nsxService.getPublicVlanId(vpc.getZoneId(), vpc.getAccountId(), + vpc.getDomainId(), vpc.getId(), null); + if (publicVlanId == null) { + allocatedIp = ipAddressManager.allocateIp(owner, false, CallContext.current().getCallingAccount(), + CallContext.current().getCallingUser(), zone, null, null); + vpcService.associateIPToVpc(allocatedIp.getId(), vpc.getId()); + } else { + allocatedIp = ipAddressManager.assignDedicateIpAddressFromNsxVrfPublicRange(owner, null, + vpc.getId(), zone.getId(), false, publicVlanId); + } + userIpAddressDetailsDao.addDetail(allocatedIp.getId(), NSX_VPN_GATEWAY_IP_DETAIL, "true", false); + IPAddressVO ip = ipAddressDao.findById(allocatedIp.getId()); + if (ip == null) { + throw new CloudRuntimeException(String.format("The allocated VPN gateway IP %s could not be loaded after association", + allocatedIp.getId())); + } + if (ip.isSourceNat()) { + throw new CloudRuntimeException(String.format( + "The allocated IP %s became a source NAT IP when it was associated to VPC %s; it cannot be used as a dedicated VPN endpoint", + ip.getAddress(), vpc.getName())); + } + nsxService.validatePublicIpVlan(vpc.getZoneId(), vpc.getAccountId(), vpc.getDomainId(), + vpc.getId(), null, ip.getVlanId()); + return ip; + } catch (Exception e) { + // do not leak the IP when associating or tagging it fails after allocation succeeded + if (Objects.nonNull(allocatedIp)) { + IPAddressVO ipToRelease = ipAddressDao.findById(allocatedIp.getId()); + if (Objects.nonNull(ipToRelease)) { + try { + releaseAutoAcquiredVpnGatewayIp(ipToRelease); + } catch (Exception releaseException) { + logger.warn("Failed to release the IP {} allocated for the VPN gateway of VPC {}: {}", + ipToRelease.getAddress().addr(), vpc.getName(), releaseException.getMessage()); + } + } else { + try { + ipAddressManager.disassociatePublicIpAddress(allocatedIp, CallContext.current().getCallingUserId(), + CallContext.current().getCallingAccount()); + } catch (Exception releaseException) { + logger.warn("Failed to release allocated VPN gateway IP {} of VPC {} after its database row disappeared: {}", + allocatedIp.getId(), vpc.getName(), releaseException.getMessage()); + } + } + } + throw new CloudRuntimeException(String.format("Failed to acquire an IP for the VPN gateway of VPC %s: %s", + vpc.getName(), e.getMessage()), e); + } + } + + private void releaseAutoAcquiredVpnGatewayIp(IPAddressVO ip) { + boolean disassociated = ipAddressManager.disassociatePublicIpAddress(ip, CallContext.current().getCallingUserId(), + CallContext.current().getCallingAccount()); + if (!disassociated) { + throw new CloudRuntimeException(String.format("Failed to disassociate auto-acquired VPN gateway IP %s", ip.getAddress())); + } + userIpAddressDetailsDao.removeDetail(ip.getId(), NSX_VPN_GATEWAY_IP_DETAIL); + } + + @Override + public void releaseVpnGatewayIp(Site2SiteVpnGateway gateway) { + VpcVO vpc = vpcDao.findById(gateway.getVpcId()); + if (vpc != null) { + try { + if (!nsxService.deleteVpnGateway(vpc)) { + throw new CloudRuntimeException(String.format("The NSX VPN gateway service for VPC %s was not deleted: the provider returned an unsuccessful answer", + vpc.getName())); + } + } catch (Exception e) { + throw new CloudRuntimeException(String.format("Failed to delete the NSX VPN gateway of VPC %s: %s", + vpc.getName(), e.getMessage()), e); + } + } + IPAddressVO ip = ipAddressDao.findById(gateway.getAddrId()); + if (Objects.isNull(ip)) { + return; + } + UserIpAddressDetailVO autoAcquiredDetail = userIpAddressDetailsDao.findDetail(ip.getId(), NSX_VPN_GATEWAY_IP_DETAIL); + if (Objects.isNull(autoAcquiredDetail)) { + return; + } + if (Boolean.parseBoolean(autoAcquiredDetail.getValue())) { + logger.debug("Releasing the auto-acquired VPN gateway IP {} of VPC {}", ip.getAddress().addr(), + vpc == null ? gateway.getVpcId() : vpc.getName()); + releaseAutoAcquiredVpnGatewayIp(ip); + } else { + // The marker is provider ownership state, not a permanent attribute of the address. + // Remove it after the NSX objects are gone so a later gateway using this IP cannot be + // mistaken for an NSX-owned gateway during offering-change cleanup. + userIpAddressDetailsDao.removeDetail(ip.getId(), NSX_VPN_GATEWAY_IP_DETAIL); + } + } + + @Override + public boolean ownsVpnGateway(Site2SiteVpnGateway gateway) { + if (gateway == null) { + return false; + } + return userIpAddressDetailsDao.findDetail(gateway.getAddrId(), NSX_VPN_GATEWAY_IP_DETAIL) != null; + } + + @Override + public void validateSite2SiteVpnCustomerGateway(Site2SiteCustomerGateway customerGateway) { + if (!NetUtils.isValidIp4(customerGateway.getGatewayIp())) { + throw new InvalidParameterValueException(String.format( + "NSX Site-to-Site VPN requires an IPv4 peer address; customer gateway %s uses %s", + customerGateway.getName(), customerGateway.getGatewayIp())); + } + NsxVpnCryptoUtils.validate(customerGateway.getIkePolicy(), customerGateway.getEspPolicy(), + customerGateway.getIkeVersion(), customerGateway.getIkeLifetime(), customerGateway.getEspLifetime(), + customerGateway.getIpsecPsk()); + } + + @Override + public boolean startSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException { + Site2SiteVpnGatewayVO vpnGateway = vpnGatewayDao.findById(conn.getVpnGatewayId()); + if (Objects.isNull(vpnGateway)) { + throw new CloudRuntimeException(String.format( + "Cannot find the VPN gateway %s of the Site-to-Site VPN connection %s", conn.getVpnGatewayId(), conn.getUuid())); + } + VpcVO vpc = vpcDao.findById(vpnGateway.getVpcId()); + if (Objects.isNull(vpc)) { + throw new CloudRuntimeException(String.format( + "Cannot find the VPC %s of the VPN gateway of Site-to-Site VPN connection %s", + vpnGateway.getVpcId(), conn.getUuid())); + } + if (!isVpnProvidedByNsx(vpc, vpnGateway)) { + return true; + } + Site2SiteCustomerGatewayVO customerGateway = customerGatewayDao.findById(conn.getCustomerGatewayId()); + if (Objects.isNull(customerGateway)) { + throw new CloudRuntimeException(String.format( + "Cannot find the customer gateway %s of the Site-to-Site VPN connection %s", conn.getCustomerGatewayId(), conn.getUuid())); + } + validateSite2SiteVpnCustomerGateway(customerGateway); + if (BooleanUtils.isTrue(customerGateway.getEncap())) { + logger.debug("Ignoring forceencap for the NSX VPN connection {}: NSX negotiates NAT-T automatically", conn); + } + if (BooleanUtils.isTrue(customerGateway.getSplitConnections())) { + logger.debug("Ignoring splitconnections for the NSX VPN connection {}: a route-based session carries all subnets", conn); + } + IPAddressVO localEndpointIp = ipAddressDao.findById(vpnGateway.getAddrId()); + if (Objects.isNull(localEndpointIp)) { + throw new CloudRuntimeException(String.format( + "Cannot find the local endpoint IP %s of the VPN gateway of VPC %s", vpnGateway.getAddrId(), vpc.getName())); + } + Pair vtiAddresses = NsxHelper.getVpnVtiAddressPair(conn.getId()); + List peerCidrs = Arrays.stream(customerGateway.getGuestCidrList().split(",")) + .map(String::trim) + .collect(Collectors.toList()); + return nsxService.createVpnConnection(vpc, conn.getUuid(), customerGateway.getGatewayIp(), + customerGateway.getIpsecPsk(), customerGateway.getIkePolicy(), customerGateway.getEspPolicy(), + customerGateway.getIkeLifetime(), customerGateway.getEspLifetime(), + BooleanUtils.isTrue(customerGateway.getDpd()), customerGateway.getIkeVersion(), conn.isPassive(), + peerCidrs, vtiAddresses.first(), vtiAddresses.second(), NsxHelper.VPN_VTI_PREFIX_LENGTH, + localEndpointIp.getAddress().addr()); + } + + @Override + public boolean stopSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException { + Site2SiteVpnGatewayVO vpnGateway = vpnGatewayDao.findById(conn.getVpnGatewayId()); + if (Objects.isNull(vpnGateway)) { + throw new CloudRuntimeException(String.format( + "Cannot find the VPN gateway %s of the Site-to-Site VPN connection %s", conn.getVpnGatewayId(), conn.getUuid())); + } + VpcVO vpc = vpcDao.findById(vpnGateway.getVpcId()); + if (Objects.isNull(vpc)) { + throw new CloudRuntimeException(String.format( + "Cannot find the VPC %s of the VPN gateway of Site-to-Site VPN connection %s", + vpnGateway.getVpcId(), conn.getUuid())); + } + if (!isVpnProvidedByNsx(vpc, vpnGateway)) { + return true; + } + return nsxService.updateVpnConnectionState(vpc, conn.getUuid(), false); + } + + @Override + public boolean deleteSite2SiteVpn(Site2SiteVpnConnection conn) throws ResourceUnavailableException { + Site2SiteVpnGatewayVO vpnGateway = vpnGatewayDao.findById(conn.getVpnGatewayId()); + if (Objects.isNull(vpnGateway)) { + throw new CloudRuntimeException(String.format( + "Cannot find the VPN gateway %s of the Site-to-Site VPN connection %s", conn.getVpnGatewayId(), conn.getUuid())); + } + VpcVO vpc = vpcDao.findById(vpnGateway.getVpcId()); + if (vpc == null) { + return true; + } + return nsxService.deleteVpnConnection(vpc, conn.getUuid()); + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java index 0f7865d7d73f..3fd90064253f 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxGuestNetworkGuru.java @@ -36,6 +36,7 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.guru.GuestNetworkGuru; +import com.cloud.network.nsx.NsxService; import com.cloud.network.vpc.VpcVO; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.NetworkOfferingVO; @@ -49,7 +50,6 @@ import org.apache.cloudstack.NsxAnswer; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; -import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; import org.apache.cloudstack.utils.NsxControllerUtils; @@ -59,6 +59,7 @@ import javax.inject.Inject; import java.util.List; +import java.util.Map; import java.util.Objects; public class NsxGuestNetworkGuru extends GuestNetworkGuru implements NetworkMigrationResponder { @@ -68,6 +69,8 @@ public class NsxGuestNetworkGuru extends GuestNetworkGuru implements NetworkMigr NsxControllerUtils nsxControllerUtils; @Inject NetworkModel networkModel; + @Inject + NsxService nsxService; public NsxGuestNetworkGuru() { super(); @@ -228,6 +231,8 @@ public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfil long dataCenterId = zone.getId(); long resourceId = network.getId(); PublicIpAddress ipAddress = networkModel.getSourceNatIpAddressForGuestNetwork(account, network); + nsxService.validatePublicIpVlan(dataCenterId, accountId, domainId, null, resourceId, + ipAddress.getVlanId()); String translatedIp = ipAddress.getAddress().addr(); String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(domainId, accountId, dataCenterId, resourceId, false); logger.debug("Creating NSX NAT Rule for Tier1 GW {} for translated IP {} for Isolated network {}", tier1GatewayName, translatedIp, network); @@ -315,19 +320,26 @@ public void createNsxSegment(NetworkVO networkVO, DataCenter zone) { NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(networkOfferingId); boolean isSourceNatSupported = !NetworkOffering.NetworkMode.ROUTED.equals(networkOfferingVO.getNetworkMode()) && networkOfferingServiceMapDao.areServicesSupportedByNetworkOffering(networkVO.getNetworkOfferingId(), Network.Service.SourceNat); - CreateNsxTier1GatewayCommand nsxTier1GatewayCommand = new CreateNsxTier1GatewayCommand(domain.getId(), account.getId(), zone.getId(), networkVO.getId(), networkVO.getName(), false, isSourceNatSupported); - - NsxAnswer nsxAnswer = nsxControllerUtils.sendNsxCommand(nsxTier1GatewayCommand, zone.getId()); - if (!nsxAnswer.getResult()) { - String msg = String.format("Could not create a Tier 1 Gateway for network %s: %s", networkVO, nsxAnswer.getDetails()); + if (!nsxService.createNetwork(zone.getId(), account.getId(), domain.getId(), networkVO.getId(), + networkVO.getName(), isSourceNatSupported)) { + String msg = String.format("Could not create a Tier 1 Gateway for network %s", networkVO); logger.error(msg); throw new CloudRuntimeException(msg); } } - CreateNsxSegmentCommand command = NsxHelper.createNsxSegmentCommand(domain, account, zone, vpcName, networkVO); + Map offeringDetails = _networkModel.getNtwkOffDetails(networkVO.getNetworkOfferingId()); + String ipDiscoveryProfile = getOfferingDetail(offeringDetails, NetworkOffering.Detail.NsxIpDiscoveryProfileId); + String macDiscoveryProfile = getOfferingDetail(offeringDetails, NetworkOffering.Detail.NsxMacDiscoveryProfileId); + String segmentSecurityProfile = getOfferingDetail(offeringDetails, NetworkOffering.Detail.NsxSegmentSecurityProfileId); + CreateNsxSegmentCommand command = NsxHelper.createNsxSegmentCommand(domain, account, zone, vpcName, networkVO, + ipDiscoveryProfile, macDiscoveryProfile, segmentSecurityProfile); NsxAnswer answer = nsxControllerUtils.sendNsxCommand(command, zone.getId()); if (!answer.getResult()) { throw new CloudRuntimeException("can not create NSX network"); } } + + protected String getOfferingDetail(Map offeringDetails, NetworkOffering.Detail detail) { + return offeringDetails == null ? null : offeringDetails.get(detail); + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderService.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderService.java index 47dfe04db3ef..4c48fefb2ed6 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderService.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderService.java @@ -20,7 +20,11 @@ import com.cloud.utils.component.PluggableService; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.command.AddNsxControllerCmd; +import org.apache.cloudstack.api.command.AddNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.AssignNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.ListNsxVrfGatewaysCmd; import org.apache.cloudstack.api.response.NsxControllerResponse; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; import java.util.List; @@ -32,4 +36,18 @@ public interface NsxProviderService extends PluggableService { List listNsxProviders(Long zoneId); boolean deleteNsxController(Long nsxControllerId); + + /** Registers a tier-0 gateway the operator has already staged in NSX. */ + NsxVrfGatewayResponse addNsxVrfGateway(AddNsxVrfGatewayCmd cmd); + + /** Claims a registered gateway for an account or a domain. */ + NsxVrfGatewayResponse assignNsxVrfGateway(AssignNsxVrfGatewayCmd cmd); + + /** Returns a gateway to the unassigned pool. */ + NsxVrfGatewayResponse releaseNsxVrfGateway(Long id); + + List listNsxVrfGateways(ListNsxVrfGatewaysCmd cmd); + + /** Removes CloudStack's registration; the gateway stays in NSX. */ + boolean deleteNsxVrfGateway(Long id); } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderServiceImpl.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderServiceImpl.java index c59ebfd87551..95ea1d0e2281 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderServiceImpl.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxProviderServiceImpl.java @@ -17,16 +17,29 @@ package org.apache.cloudstack.service; import com.amazonaws.util.CollectionUtils; +import com.cloud.dc.AccountVlanMapVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DomainVlanMapVO; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanDetailsVO; +import com.cloud.dc.VlanVO; +import com.cloud.dc.dao.AccountVlanMapDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.DomainVlanMapDao; +import com.cloud.dc.dao.VlanDao; +import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.dao.HostDetailsDao; import com.cloud.network.Network; +import com.cloud.network.IpAddress; import com.cloud.network.Networks; import com.cloud.network.nsx.NsxProvider; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkServiceMapDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; @@ -40,12 +53,35 @@ import org.apache.cloudstack.api.command.DeleteNsxControllerCmd; import org.apache.cloudstack.api.command.ListNsxControllersCmd; import org.apache.cloudstack.api.BaseResponse; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.api.command.AddNsxControllerCmd; +import org.apache.cloudstack.api.command.AddNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.AssignNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.DeleteNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.ListNsxVrfGatewaysCmd; +import org.apache.cloudstack.api.command.ReleaseNsxVrfGatewayCmd; +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; +import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.NsxVrfGatewayValidationAnswer; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; import org.apache.cloudstack.api.response.NsxControllerResponse; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.resource.NsxResource; +import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import javax.inject.Inject; import javax.naming.ConfigurationException; import java.util.ArrayList; @@ -57,6 +93,8 @@ public class NsxProviderServiceImpl implements NsxProviderService { + protected Logger logger = LogManager.getLogger(getClass()); + @Inject NsxProviderDao nsxProviderDao; @Inject @@ -66,9 +104,37 @@ public class NsxProviderServiceImpl implements NsxProviderService { @Inject NetworkDao networkDao; @Inject + NetworkServiceMapDao networkServiceMapDao; + @Inject ResourceManager resourceManager; @Inject HostDetailsDao hostDetailsDao; + @Inject + NsxVrfGatewayDao nsxVrfGatewayDao; + @Inject + IPAddressDao ipAddressDao; + @Inject + NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Inject + AccountDao accountDao; + @Inject + DomainDao domainDao; + @Inject + VpcDao vpcDao; + @Inject + VpcOfferingServiceMapDao vpcOfferingServiceMapDao; + @Inject + VlanDao vlanDao; + @Inject + VlanDetailsDao vlanDetailsDao; + @Inject + AccountVlanMapDao accountVlanMapDao; + @Inject + DomainVlanMapDao domainVlanMapDao; + @Inject + NsxControllerUtils nsxControllerUtils; + @Inject + NsxVrfGatewayLockManager nsxVrfGatewayLockManager; @Override public NsxProvider addProvider(AddNsxControllerCmd cmd) { @@ -197,10 +263,356 @@ public List> getCommands() { cmdList.add(AddNsxControllerCmd.class); cmdList.add(ListNsxControllersCmd.class); cmdList.add(DeleteNsxControllerCmd.class); + cmdList.add(AddNsxVrfGatewayCmd.class); + cmdList.add(AssignNsxVrfGatewayCmd.class); + cmdList.add(ReleaseNsxVrfGatewayCmd.class); + cmdList.add(ListNsxVrfGatewaysCmd.class); + cmdList.add(DeleteNsxVrfGatewayCmd.class); } return cmdList; } + @Override + public NsxVrfGatewayResponse addNsxVrfGateway(AddNsxVrfGatewayCmd cmd) { + DataCenterVO zone = dataCenterDao.findById(cmd.getZoneId()); + if (zone == null) { + throw new InvalidParameterValueException("Could not find zone with the given ID"); + } + NsxProviderVO provider = nsxProviderDao.findByZoneId(zone.getId()); + if (provider == null) { + throw new InvalidParameterValueException(String.format("Zone %s has no NSX controller registered", zone)); + } + validateDeclaredParent(provider, cmd.getParentTier0Gateway()); + + ValidateNsxVrfGatewayCommand validateCommand = new ValidateNsxVrfGatewayCommand(zone.getId(), + cmd.getTier0Gateway(), cmd.getParentTier0Gateway(), cmd.getEdgeCluster()); + NsxAnswer answer = nsxControllerUtils.sendNsxCommandForResult(validateCommand, zone.getId()); + if (!(answer instanceof NsxVrfGatewayValidationAnswer) || !answer.getResult()) { + String details = answer == null ? "no answer was returned" : answer.getDetails(); + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway validation failed: %s", details)); + } + NsxVrfGatewayValidationAnswer validation = (NsxVrfGatewayValidationAnswer) answer; + + return nsxVrfGatewayLockManager.withZoneLock(zone.getId(), () -> + Transaction.execute((TransactionCallback) status -> { + VlanVO publicVlan = validatePublicVlan(zone.getId(), cmd.getPublicVlanId()); + ipAddressDao.lockRange(publicVlan.getId()); + publicVlan = validatePublicVlan(zone.getId(), publicVlan.getId()); + if (nsxVrfGatewayDao.findByZoneAndTier0Name(zone.getId(), cmd.getTier0Gateway()) != null) { + throw new InvalidParameterValueException(String.format( + "Tier-0 gateway %s is already registered in zone %s", cmd.getTier0Gateway(), zone)); + } + if (nsxVrfGatewayDao.findByPublicVlan(publicVlan.getId()) != null) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s is already registered to another NSX VRF gateway", publicVlan)); + } + + NsxVrfGatewayVO gateway = new NsxVrfGatewayVO(zone.getId(), cmd.getTier0Gateway(), + validation.getEdgeClusterPath(), validation.getParentTier0Path()); + gateway.setPublicVlanDbId(publicVlan.getId()); + gateway = nsxVrfGatewayDao.persist(gateway); + if (gateway == null) { + throw new CloudRuntimeException(String.format( + "Failed to register NSX VRF gateway %s in zone %s", cmd.getTier0Gateway(), zone)); + } + logger.debug("Registered NSX VRF gateway {} in zone {}", gateway, zone); + return createNsxVrfGatewayResponse(gateway); + })); + } + + @Override + public NsxVrfGatewayResponse assignNsxVrfGateway(AssignNsxVrfGatewayCmd cmd) { + NsxVrfGatewayVO requestedGateway = getVrfGatewayOrFail(cmd.getId()); + Long accountId = cmd.getAccountId(); + Long domainId = cmd.getDomainId(); + + if ((accountId == null) == (domainId == null)) { + throw new InvalidParameterValueException("Specify exactly one of accountid or domainid"); + } + + return nsxVrfGatewayLockManager.withZoneLock(requestedGateway.getZoneId(), () -> { + NsxVrfGatewayVO gateway = getVrfGatewayOrFail(cmd.getId()); + if (!gateway.isUnclaimed()) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s is already assigned; release it first", gateway.getNsxTier0Name())); + } + + AccountVO account = accountId == null ? null : accountDao.findById(accountId); + if (accountId != null && account == null) { + throw new InvalidParameterValueException("Could not find account with the given ID"); + } + DomainVO domain = domainId == null ? null : domainDao.findById(domainId); + if (domainId != null && domain == null) { + throw new InvalidParameterValueException("Could not find domain with the given ID"); + } + assertNoExistingTenantPlacement(gateway, accountId, domainId); + validatePublicRangeDedication(gateway, accountId, domainId); + + if (accountId != null) { + NsxVrfGatewayVO existing = nsxVrfGatewayDao.findByAccount(gateway.getZoneId(), accountId); + if (existing != null) { + throw new InvalidParameterValueException(String.format( + "Account %s already has NSX VRF gateway %s in this zone", account, existing.getNsxTier0Name())); + } + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(accountId); + gateway.setDomainId(null); + } else { + NsxVrfGatewayVO existing = nsxVrfGatewayDao.findByDomain(gateway.getZoneId(), domainId); + if (existing != null) { + throw new InvalidParameterValueException(String.format( + "Domain %s already has NSX VRF gateway %s in this zone", domain, existing.getNsxTier0Name())); + } + gateway.setScope(NsxVrfGatewayVO.Scope.DOMAIN.name()); + gateway.setAccountId(null); + gateway.setDomainId(domainId); + } + + if (!nsxVrfGatewayDao.update(gateway.getId(), gateway)) { + throw new CloudRuntimeException(String.format( + "Failed to assign NSX VRF gateway %s", gateway.getNsxTier0Name())); + } + logger.debug("Assigned NSX VRF gateway {}", gateway); + return createNsxVrfGatewayResponse(gateway); + }); + } + + @Override + public NsxVrfGatewayResponse releaseNsxVrfGateway(Long id) { + NsxVrfGatewayVO requestedGateway = getVrfGatewayOrFail(id); + return nsxVrfGatewayLockManager.withZoneLock(requestedGateway.getZoneId(), () -> { + NsxVrfGatewayVO gateway = getVrfGatewayOrFail(id); + if (gateway.isUnclaimed()) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s is not assigned to anyone", gateway.getNsxTier0Name())); + } + long placements = nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId()); + if (placements > 0) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s still has %d Tier-1 placement(s); remove them before releasing it", + gateway.getNsxTier0Name(), placements)); + } + gateway.setScope(null); + gateway.setAccountId(null); + gateway.setDomainId(null); + if (!nsxVrfGatewayDao.update(gateway.getId(), gateway)) { + throw new CloudRuntimeException(String.format( + "Failed to release NSX VRF gateway %s", gateway.getNsxTier0Name())); + } + logger.debug("Released NSX VRF gateway {}", gateway); + return createNsxVrfGatewayResponse(gateway); + }); + } + + @Override + public List listNsxVrfGateways(ListNsxVrfGatewaysCmd cmd) { + List gateways; + if (cmd.getAccountId() != null) { + gateways = new ArrayList<>(); + for (DataCenterVO zone : listZonesToSearch(cmd.getZoneId())) { + NsxVrfGatewayVO gateway = nsxVrfGatewayDao.findByAccount(zone.getId(), cmd.getAccountId()); + if (gateway != null) { + gateways.add(gateway); + } + } + } else if (cmd.getDomainId() != null) { + gateways = new ArrayList<>(); + for (DataCenterVO zone : listZonesToSearch(cmd.getZoneId())) { + NsxVrfGatewayVO gateway = nsxVrfGatewayDao.findByDomain(zone.getId(), cmd.getDomainId()); + if (gateway != null) { + gateways.add(gateway); + } + } + } else if (cmd.getZoneId() != null) { + gateways = nsxVrfGatewayDao.listByZone(cmd.getZoneId()); + } else { + gateways = nsxVrfGatewayDao.listAll(); + } + + List responses = new ArrayList<>(); + for (NsxVrfGatewayVO gateway : gateways) { + if (cmd.getAllocatedOnly() != null && cmd.getAllocatedOnly() == gateway.isUnclaimed()) { + continue; + } + responses.add(createNsxVrfGatewayResponse(gateway)); + } + return responses; + } + + @Override + public boolean deleteNsxVrfGateway(Long id) { + NsxVrfGatewayVO requestedGateway = getVrfGatewayOrFail(id); + return nsxVrfGatewayLockManager.withZoneLock(requestedGateway.getZoneId(), () -> { + NsxVrfGatewayVO gateway = getVrfGatewayOrFail(id); + if (!gateway.isUnclaimed()) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s is still assigned; release it first", gateway.getNsxTier0Name())); + } + if (nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId()) > 0) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s still has Tier-1 placements", gateway.getNsxTier0Name())); + } + logger.debug("Deregistering NSX VRF gateway {}; it is left in place in NSX", gateway); + return nsxVrfGatewayDao.expunge(id); + }); + } + + private List listZonesToSearch(Long zoneId) { + if (zoneId == null) { + return dataCenterDao.listAll(); + } + DataCenterVO zone = dataCenterDao.findById(zoneId); + return zone == null ? new ArrayList<>() : List.of(zone); + } + + private NsxVrfGatewayVO getVrfGatewayOrFail(Long id) { + NsxVrfGatewayVO gateway = id == null ? null : nsxVrfGatewayDao.findById(id); + if (gateway == null) { + throw new InvalidParameterValueException("Could not find NSX VRF gateway with the given ID"); + } + return gateway; + } + + private VlanVO validatePublicVlan(long zoneId, Long vlanId) { + VlanVO vlan = vlanId == null ? null : vlanDao.findById(vlanId); + if (vlan == null || vlan.getDataCenterId() != zoneId || vlan.getVlanType() != Vlan.VlanType.VirtualNetwork) { + throw new InvalidParameterValueException("The public IP range is not a virtual-network range in the selected zone"); + } + VlanDetailsVO nsxDetail = vlanDetailsDao.findDetail(vlan.getId(), ApiConstants.NSX_DETAIL_KEY); + if (nsxDetail == null || !Boolean.parseBoolean(nsxDetail.getValue())) { + throw new InvalidParameterValueException("The public IP range is not enabled for the zone's NSX provider"); + } + List publicIps = ipAddressDao.listByVlanId(vlan.getId()); + if (publicIps.stream().anyMatch(IPAddressVO::isForSystemVms)) { + throw new InvalidParameterValueException( + "The public IP range registered to an NSX VRF gateway cannot be reserved for system VMs"); + } + if (publicIps.stream() + .anyMatch(ip -> ip.getState() != IpAddress.State.Free)) { + throw new InvalidParameterValueException( + "The public IP range must not contain allocated or reserved addresses when it is registered to an NSX VRF gateway"); + } + return vlan; + } + + private void validateDeclaredParent(NsxProviderVO provider, String parentTier0) { + if (StringUtils.isBlank(parentTier0)) { + return; + } + String configuredTier0 = provider.getTier0Gateway(); + if (!parentTier0.equals(configuredTier0) && !parentTier0.endsWith("/" + configuredTier0)) { + throw new InvalidParameterValueException(String.format( + "The declared VRF parent %s is not the zone Tier-0 %s", parentTier0, configuredTier0)); + } + } + + private void validatePublicRangeDedication(NsxVrfGatewayVO gateway, Long accountId, Long domainId) { + Long vlanId = gateway.getPublicVlanDbId(); + if (vlanId == null) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s has no public IP range", gateway.getNsxTier0Name())); + } + if (accountId != null) { + List mappings = accountVlanMapDao.listAccountVlanMapsByVlan(vlanId); + boolean exact = mappings.size() == 1 && mappings.get(0).getAccountId() == accountId; + if (!exact || !domainVlanMapDao.listDomainVlanMapsByVlan(vlanId).isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s must be dedicated to the account before assigning gateway %s", + vlanId, gateway.getNsxTier0Name())); + } + return; + } + List mappings = domainVlanMapDao.listDomainVlanMapsByVlan(vlanId); + boolean exact = mappings.size() == 1 && mappings.get(0).getDomainId() == domainId; + if (!exact || !accountVlanMapDao.listAccountVlanMapsByVlan(vlanId).isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s must be dedicated to the domain before assigning gateway %s", + vlanId, gateway.getNsxTier0Name())); + } + } + + private void assertNoExistingTenantPlacement(NsxVrfGatewayVO gateway, Long accountId, Long domainId) { + boolean placementExists = nsxVrfGatewayPlacementDao.listByZone(gateway.getZoneId()).stream() + .anyMatch(placement -> isTenantResource(accountId, domainId, placement.getAccountId(), + placement.getDomainId())); + if (placementExists) { + throw new InvalidParameterValueException( + "The tenant already has an NSX Tier-1 placement in this zone; existing Tier-1 gateways cannot be re-parented"); + } + + boolean legacyVpcExists = vpcDao.listByZone(gateway.getZoneId()).stream() + .filter(vpc -> isTenantResource(accountId, domainId, vpc.getAccountId(), vpc.getDomainId())) + .filter(vpc -> isNsxTier1Offering(vpc.getVpcOfferingId())) + .anyMatch(vpc -> nsxVrfGatewayPlacementDao.findByVpcId(vpc.getId()) == null); + boolean legacyNetworkExists = networkDao.listByZone(gateway.getZoneId()).stream() + .filter(network -> network.getVpcId() == null) + .filter(network -> network.getBroadcastDomainType() == Networks.BroadcastDomainType.NSX) + .filter(network -> isTenantResource(accountId, domainId, network.getAccountId(), + network.getDomainId())) + .anyMatch(network -> nsxVrfGatewayPlacementDao.findByNetworkId(network.getId()) == null); + if (legacyVpcExists || legacyNetworkExists) { + throw new InvalidParameterValueException( + "The tenant has existing NSX Tier-1 resources without recorded placement; backfill their current Tier-0 placement before assigning a VRF gateway"); + } + } + + private boolean isTenantResource(Long accountId, Long domainId, long resourceAccountId, long resourceDomainId) { + return accountId != null ? accountId == resourceAccountId + : domainDao.isChildDomain(domainId, resourceDomainId); + } + + private boolean isNsxTier1Offering(long offeringId) { + return vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.SourceNat.getName(), + Network.Provider.Nsx.getName(), offeringId) != null + || vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.Gateway.getName(), + Network.Provider.Nsx.getName(), offeringId) != null; + } + + @VisibleForTesting + long countNetworksUsingVrfGateway(NsxVrfGatewayVO gateway) { + return nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId()); + } + + @VisibleForTesting + NsxVrfGatewayResponse createNsxVrfGatewayResponse(NsxVrfGatewayVO gateway) { + NsxVrfGatewayResponse response = new NsxVrfGatewayResponse(); + response.setId(gateway.getUuid()); + response.setTier0Gateway(gateway.getNsxTier0Name()); + response.setEdgeCluster(gateway.getEdgeCluster()); + response.setParentTier0Gateway(gateway.getParentTier0()); + response.setScope(gateway.getScope()); + response.setAllocated(!gateway.isUnclaimed()); + + if (gateway.getPublicVlanDbId() != null) { + VlanVO vlan = vlanDao.findById(gateway.getPublicVlanDbId()); + if (vlan != null) { + response.setPublicVlanId(vlan.getUuid()); + } + } + + DataCenterVO zone = dataCenterDao.findById(gateway.getZoneId()); + if (zone != null) { + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + } + if (gateway.getAccountId() != null) { + AccountVO account = accountDao.findById(gateway.getAccountId()); + if (account != null) { + response.setAccountId(account.getUuid()); + response.setAccountName(account.getAccountName()); + } + } + if (gateway.getDomainId() != null) { + DomainVO domain = domainDao.findById(gateway.getDomainId()); + if (domain != null) { + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); + } + } + return response; + } + @VisibleForTesting void validateNetworkState(List networkList) { for (NetworkVO network : networkList) { diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java index 9c631ac9a6dc..aa679fa5b4cc 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxPublicNetworkGuru.java @@ -106,10 +106,18 @@ public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfil logger.error(err); throw new CloudRuntimeException(err); } - ips = ips.stream().filter(x -> !x.getAddress().addr().equals(nic.getIPv4Address())).collect(Collectors.toList()); + ips = ips.stream() + .filter(IPAddressVO::isSourceNat) + .filter(x -> !x.isForSystemVms()) + .filter(x -> !x.getAddress().addr().equals(nic.getIPv4Address())) + .collect(Collectors.toList()); + if (ips.size() != 1) { + throw new CloudRuntimeException(String.format( + "Expected one tenant source NAT IP for VPC %s, found %d", vpc, ips.size())); + } // Use Source NAT IP address from the NSX Public Range. Do not Use the VR Public IP address ipAddress = ips.get(0); - if (ipAddress.isSourceNat() && !ipAddress.isForSystemVms()) { + if (ipAddress.isSourceNat()) { VlanDetailsVO detail = vlanDetailsDao.findDetail(ipAddress.getVlanId(), ApiConstants.NSX_DETAIL_KEY); if (detail != null && detail.getValue().equalsIgnoreCase("true")) { long accountId = vpc.getAccountId(); @@ -123,7 +131,8 @@ public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfil vpcOfferingServiceMapDao.areServicesSupportedByVpcOffering(vpc.getVpcOfferingId(), services); logger.info("Creating Tier 1 Gateway for VPC {}", vpc); - boolean result = nsxService.createVpcNetwork(dataCenterId, accountId, domainId, resourceId, vpc.getName(), sourceNatEnabled); + boolean result = nsxService.createVpcNetwork(dataCenterId, accountId, domainId, resourceId, + vpc.getName(), sourceNatEnabled, ipAddress.getVlanId()); if (!result) { String msg = String.format("Error creating Tier 1 Gateway for VPC %s", vpc); logger.error(msg); diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java index d48def5c9bd5..8fdb0409cb53 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java @@ -16,54 +16,411 @@ // under the License. package org.apache.cloudstack.service; +import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import javax.naming.ConfigurationException; import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.NsxVrfGatewayValidationAnswer; import org.apache.cloudstack.agent.api.CreateNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.CreateNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnGatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnGatewayCommand; +import org.apache.cloudstack.agent.api.GetNsxVpnSessionStatusCommand; +import org.apache.cloudstack.agent.api.UpdateNsxVpnConnectionStateCommand; +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.resource.NsxNetworkRule; +import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.cloudstack.utils.NsxHelper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.cloud.alert.AlertManager; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.SDNProviderNetworkRule; +import com.cloud.network.Site2SiteVpnConnection; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; +import com.cloud.network.dao.Site2SiteVpnConnectionDao; +import com.cloud.network.dao.Site2SiteVpnConnectionVO; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.Site2SiteVpnGatewayVO; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.element.NsxVrfGatewayPlacementVO; +import com.cloud.network.element.NsxProviderVO; import com.cloud.network.nsx.NsxService; +import com.cloud.network.nsx.NsxVpnGatewayResult; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.StringUtils; + +public class NsxServiceImpl extends ManagerBase implements NsxService, Configurable { + + public static final ConfigKey NSX_VPN_STATUS_POLL_INTERVAL = new ConfigKey<>("Advanced", Integer.class, + "nsx.vpn.status.poll.interval", "60", + "Interval (in seconds) between two NSX Site-to-Site VPN connection status polls; requires a management server restart", + false, ConfigKey.Scope.Global); + + public static final ConfigKey NSX_VRF_SCOPE = new ConfigKey<>(String.class, + "nsx.vrf.scope", "Advanced", "NONE", + "Routing domain granularity for NSX zones. NONE (default) attaches every tenant's Tier-1 " + + "gateways to the zone-wide Tier-0. ACCOUNT or DOMAIN attaches them to the registered " + + "tenant-specific Tier-0 or VRF gateway", + true, ConfigKey.Scope.Zone, null, null, null, null, null, + ConfigKey.Kind.Select, "NONE,ACCOUNT,DOMAIN"); + + public static final ConfigKey NSX_VRF_FALLBACK_TO_SHARED_TIER0 = new ConfigKey<>("Advanced", Boolean.class, + "nsx.vrf.fallback.to.shared.tier0", "false", + "Allow a tenant without an assigned NSX VRF gateway to use the zone-wide Tier-0 when nsx.vrf.scope is enabled", + true, ConfigKey.Scope.Zone); + + protected static final String NSX_VRF_SCOPE_NONE = "NONE"; + + protected static final String VPN_SESSION_STATUS_UP = "UP"; + protected static final String VPN_SESSION_STATUS_DOWN = "DOWN"; + protected static final String VPN_SESSION_STATUS_DEGRADED = "DEGRADED"; + protected static final String VPN_SESSION_STATUS_NOT_FOUND = "NOT_FOUND"; + protected static final int VPN_STATUS_POLL_FAILURE_THRESHOLD = 3; + protected static final int VPN_STATUS_POLL_MIN_INTERVAL = 10; + protected static final int VPN_STATUS_POLL_DEFAULT_INTERVAL = 60; + + private static final List VPN_POLLED_STATES = List.of( + Site2SiteVpnConnection.State.Pending, Site2SiteVpnConnection.State.Connecting, Site2SiteVpnConnection.State.Connected, + Site2SiteVpnConnection.State.Disconnected); -public class NsxServiceImpl implements NsxService, Configurable { @Inject NsxControllerUtils nsxControllerUtils; @Inject VpcDao vpcDao; + @Inject + Site2SiteVpnConnectionDao site2SiteVpnConnectionDao; + @Inject + Site2SiteVpnGatewayDao site2SiteVpnGatewayDao; + @Inject + NsxVrfGatewayDao nsxVrfGatewayDao; + @Inject + NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Inject + NsxProviderDao nsxProviderDao; + @Inject + NsxVrfGatewayLockManager nsxVrfGatewayLockManager; + @Inject + DomainDao domainDao; + @Inject + UserIpAddressDetailsDao userIpAddressDetailsDao; + @Inject + AlertManager alertManager; protected Logger logger = LogManager.getLogger(getClass()); - public boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled) { + private ScheduledExecutorService vpnStatusPollExecutor; + private final Map vpnStatusPollFailures = new ConcurrentHashMap<>(); + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + return true; + } + + @Override + public synchronized boolean start() { + super.start(); + if (vpnStatusPollExecutor != null && !vpnStatusPollExecutor.isShutdown()) { + return true; + } + Integer configuredInterval = NSX_VPN_STATUS_POLL_INTERVAL.value(); + int pollInterval = Objects.isNull(configuredInterval) ? VPN_STATUS_POLL_DEFAULT_INTERVAL : configuredInterval; + if (pollInterval < VPN_STATUS_POLL_MIN_INTERVAL) { + logger.warn("The configured value {} of {} is below the minimum of {} seconds, using the default of {} seconds", + configuredInterval, NSX_VPN_STATUS_POLL_INTERVAL.key(), VPN_STATUS_POLL_MIN_INTERVAL, VPN_STATUS_POLL_DEFAULT_INTERVAL); + pollInterval = VPN_STATUS_POLL_DEFAULT_INTERVAL; + } + ScheduledExecutorService executor = createVpnStatusPollExecutor(); + try { + executor.scheduleWithFixedDelay(new VpnStatusPollTask(), pollInterval, pollInterval, TimeUnit.SECONDS); + vpnStatusPollExecutor = executor; + } catch (RuntimeException e) { + executor.shutdownNow(); + throw e; + } + return true; + } + + protected ScheduledExecutorService createVpnStatusPollExecutor() { + return Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Nsx-Vpn-Status-Poll")); + } + + @Override + public synchronized boolean stop() { + ScheduledExecutorService executor = vpnStatusPollExecutor; + vpnStatusPollExecutor = null; + if (Objects.nonNull(executor)) { + executor.shutdownNow(); + } + return super.stop(); + } + + public boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, + boolean sourceNatEnabled, Long sourceNatVlanId) { CreateNsxTier1GatewayCommand createNsxTier1GatewayCommand = new CreateNsxTier1GatewayCommand(domainId, accountId, zoneId, vpcId, vpcName, true, sourceNatEnabled); - NsxAnswer result = nsxControllerUtils.sendNsxCommand(createNsxTier1GatewayCommand, zoneId); - return result.getResult(); + return createTier1GatewayWithPlacement(createNsxTier1GatewayCommand, sourceNatVlanId); + } + + private boolean createTier1GatewayWithPlacement(CreateNsxTier1GatewayCommand cmd, Long sourceNatVlanId) { + return nsxVrfGatewayLockManager.withPlacementLock(cmd.isResourceVpc(), cmd.getNetworkResourceId(), () -> { + PlacementPlan plan = nsxVrfGatewayLockManager.withZoneLock(cmd.getZoneId(), + () -> preparePlacement(cmd, sourceNatVlanId)); + try { + if (plan.gateway != null) { + validateVrfGatewayForPlacement(plan.gateway); + } + NsxAnswer result = nsxControllerUtils.sendNsxCommand(cmd, cmd.getZoneId()); + if (!result.getResult()) { + throw new CloudRuntimeException(String.format( + "Failed to create NSX Tier-1 gateway for %s: %s", + cmd.getNetworkResourceName(), result.getDetails())); + } + plan.placement.setState(NsxVrfGatewayPlacementVO.State.ACTIVE); + updatePlacement(plan.placement); + return true; + } catch (RuntimeException e) { + markPlacementFailed(plan.placement); + throw e; + } + }); + } + + private PlacementPlan preparePlacement(CreateNsxTier1GatewayCommand cmd, Long sourceNatVlanId) { + PlacementPlan plan = reservePlacement(cmd.getZoneId(), cmd.getAccountId(), cmd.getDomainId(), + cmd.isResourceVpc() ? cmd.getNetworkResourceId() : null, + cmd.isResourceVpc() ? null : cmd.getNetworkResourceId()); + NsxVrfGatewayVO gateway = plan.gateway; + if (gateway != null) { + cmd.setTier0Gateway(gateway.getNsxTier0Name()); + cmd.setEdgeCluster(gateway.getEdgeCluster()); + if (cmd.isSourceNatEnabled() && sourceNatVlanId != null + && !Objects.equals(gateway.getPublicVlanDbId(), sourceNatVlanId)) { + throw new CloudRuntimeException(String.format( + "The source NAT IP for %s must come from public IP range %s assigned to NSX VRF gateway %s", + cmd.getNetworkResourceName(), gateway.getPublicVlanDbId(), gateway.getNsxTier0Name())); + } + } else { + cmd.setTier0Gateway(plan.placement.getTier0Name()); + } + plan.placement.setState(NsxVrfGatewayPlacementVO.State.PENDING_CREATE); + updatePlacement(plan.placement); + return plan; + } + + private PlacementPlan reservePlacement(long zoneId, long accountId, long domainId, Long vpcId, Long networkId) { + NsxVrfGatewayPlacementVO placement = vpcId != null + ? nsxVrfGatewayPlacementDao.findByVpcId(vpcId) + : nsxVrfGatewayPlacementDao.findByNetworkId(networkId); + if (placement != null) { + validatePlacementOwner(placement, zoneId, accountId, domainId); + if (NsxVrfGatewayPlacementVO.State.PENDING_DELETE.name().equals(placement.getState())) { + throw new CloudRuntimeException("The NSX Tier-1 placement is pending deletion"); + } + NsxVrfGatewayVO gateway = placement.getGatewayId() == null + ? null : nsxVrfGatewayDao.findById(placement.getGatewayId()); + if (placement.getGatewayId() != null && gateway == null) { + throw new CloudRuntimeException("The NSX VRF gateway recorded for this Tier-1 placement is unavailable"); + } + return new PlacementPlan(gateway, placement); + } + + NsxVrfGatewayVO gateway = resolveVrfGateway(zoneId, accountId, domainId); + String tier0Name = gateway == null ? getSharedTier0(zoneId) : gateway.getNsxTier0Name(); + placement = new NsxVrfGatewayPlacementVO(gateway == null ? null : gateway.getId(), zoneId, domainId, + accountId, vpcId, networkId, tier0Name); + placement = nsxVrfGatewayPlacementDao.persist(placement); + if (placement == null) { + throw new CloudRuntimeException("Failed to persist the NSX Tier-1 placement"); + } + return new PlacementPlan(gateway, placement); + } + + private void validatePlacementOwner(NsxVrfGatewayPlacementVO placement, long zoneId, long accountId, + long domainId) { + if (placement.getZoneId() != zoneId || placement.getDomainId() != domainId + || placement.getAccountId() != accountId) { + throw new CloudRuntimeException("The NSX Tier-1 placement does not belong to the requested tenant"); + } + } + + private String getSharedTier0(long zoneId) { + NsxProviderVO provider = nsxProviderDao.findByZoneId(zoneId); + if (provider == null || StringUtils.isBlank(provider.getTier0Gateway())) { + throw new CloudRuntimeException(String.format("Zone %s has no configured NSX Tier-0 gateway", zoneId)); + } + return provider.getTier0Gateway(); + } + + private void validateVrfGatewayForPlacement(NsxVrfGatewayVO gateway) { + ValidateNsxVrfGatewayCommand command = new ValidateNsxVrfGatewayCommand(gateway.getZoneId(), + gateway.getNsxTier0Name(), gateway.getParentTier0(), gateway.getEdgeCluster()); + NsxAnswer answer = nsxControllerUtils.sendNsxCommandForResult(command, gateway.getZoneId()); + if (!(answer instanceof NsxVrfGatewayValidationAnswer) || !answer.getResult()) { + String details = answer == null ? "no answer was returned" : answer.getDetails(); + throw new CloudRuntimeException(String.format( + "NSX VRF gateway %s failed placement validation: %s", + gateway.getNsxTier0Name(), details)); + } + } + + private boolean deleteTier1GatewayWithPlacement(DeleteNsxTier1GatewayCommand command, Long vpcId, Long networkId) { + boolean isVpc = vpcId != null; + long resourceId = isVpc ? vpcId : networkId; + return nsxVrfGatewayLockManager.withPlacementLock(isVpc, resourceId, () -> { + NsxVrfGatewayPlacementVO placement = nsxVrfGatewayLockManager.withZoneLock(command.getZoneId(), + () -> preparePlacementForDeletion(command, vpcId, networkId)); + try { + NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, command.getZoneId()); + if (!result.getResult()) { + throw new CloudRuntimeException(String.format( + "Failed to delete NSX Tier-1 gateway: %s", result.getDetails())); + } + if (placement != null && !nsxVrfGatewayPlacementDao.expunge(placement.getId())) { + throw new CloudRuntimeException(String.format( + "Failed to remove NSX Tier-1 placement %s after backend deletion", placement.getId())); + } + return true; + } catch (RuntimeException e) { + if (placement != null) { + markPlacementFailed(placement); + } + throw e; + } + }); + } + + private NsxVrfGatewayPlacementVO preparePlacementForDeletion(DeleteNsxTier1GatewayCommand command, + Long vpcId, Long networkId) { + NsxVrfGatewayPlacementVO placement = vpcId != null + ? nsxVrfGatewayPlacementDao.findByVpcId(vpcId) + : nsxVrfGatewayPlacementDao.findByNetworkId(networkId); + if (placement != null) { + if (placement.getZoneId() != command.getZoneId() + || placement.getAccountId() != command.getAccountId() + || placement.getDomainId() != command.getDomainId()) { + throw new CloudRuntimeException("The NSX Tier-1 placement does not belong to the requested tenant"); + } + placement.setState(NsxVrfGatewayPlacementVO.State.PENDING_DELETE); + updatePlacement(placement); + } + return placement; + } + + private void updatePlacement(NsxVrfGatewayPlacementVO placement) { + placement.setUpdated(new Date()); + if (!nsxVrfGatewayPlacementDao.update(placement.getId(), placement)) { + throw new CloudRuntimeException(String.format( + "Failed to update NSX Tier-1 placement %s", placement.getId())); + } + } + + private void markPlacementFailed(NsxVrfGatewayPlacementVO placement) { + placement.setState(NsxVrfGatewayPlacementVO.State.FAILED); + placement.setUpdated(new Date()); + if (!nsxVrfGatewayPlacementDao.update(placement.getId(), placement)) { + logger.error("Failed to mark NSX Tier-1 placement {} as failed", placement.getId()); + } + } + + private static class PlacementPlan { + private final NsxVrfGatewayVO gateway; + private final NsxVrfGatewayPlacementVO placement; + + PlacementPlan(NsxVrfGatewayVO gateway, NsxVrfGatewayPlacementVO placement) { + this.gateway = gateway; + this.placement = placement; + } + } + + protected NsxVrfGatewayVO resolveVrfGateway(Long zoneId, long accountId, long domainId) { + if (zoneId == null) { + return null; + } + String scope = getVrfScope(zoneId); + if (StringUtils.isBlank(scope) || NSX_VRF_SCOPE_NONE.equalsIgnoreCase(scope)) { + return null; + } + + NsxVrfGatewayVO gateway = null; + if (NsxVrfGatewayVO.Scope.ACCOUNT.name().equalsIgnoreCase(scope)) { + gateway = nsxVrfGatewayDao.findByAccount(zoneId, accountId); + } + if (gateway == null) { + gateway = findVrfGatewayForDomainChain(zoneId, domainId); + } + if (gateway != null) { + logger.debug("Resolved NSX VRF gateway {} for account {} in zone {}", gateway, accountId, zoneId); + return gateway; + } + if (isVrfFallbackToSharedTier0Allowed(zoneId)) { + logger.warn("Account {} has no NSX VRF gateway in zone {}; using the shared Tier-0 because {} is enabled", + accountId, zoneId, NSX_VRF_FALLBACK_TO_SHARED_TIER0.key()); + return null; + } + throw new CloudRuntimeException(String.format( + "Zone %s uses %s=%s, but account %s has no NSX VRF gateway. Assign one or enable %s", + zoneId, NSX_VRF_SCOPE.key(), scope, accountId, NSX_VRF_FALLBACK_TO_SHARED_TIER0.key())); + } + + protected String getVrfScope(long zoneId) { + return NSX_VRF_SCOPE.valueIn(zoneId); + } + + protected boolean isVrfFallbackToSharedTier0Allowed(long zoneId) { + return Boolean.TRUE.equals(NSX_VRF_FALLBACK_TO_SHARED_TIER0.valueIn(zoneId)); + } + + private NsxVrfGatewayVO findVrfGatewayForDomainChain(long zoneId, long domainId) { + DomainVO domain = domainDao.findById(domainId); + while (domain != null) { + NsxVrfGatewayVO gateway = nsxVrfGatewayDao.findByDomain(zoneId, domain.getId()); + if (gateway != null) { + return gateway; + } + Long parentId = domain.getParent(); + domain = parentId == null ? null : domainDao.findById(parentId); + } + return null; } @Override @@ -76,6 +433,8 @@ public boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address) { long zoneId = vpc.getZoneId(); long vpcId = vpc.getId(); + validatePublicIpVlan(zoneId, accountId, domainId, vpcId, null, address.getVlanId()); + logger.debug("Updating the source NAT IP for NSX VPC {} to IP: {}", vpc, address.getAddress().addr()); String tier1GatewayName = NsxControllerUtils.getTier1GatewayName(domainId, accountId, zoneId, vpcId, true); String sourceNatRuleId = NsxControllerUtils.getNsxNatRuleId(domainId, accountId, zoneId, vpcId, true); @@ -88,18 +447,78 @@ public boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address) { return true; } - public boolean createNetwork(Long zoneId, long accountId, long domainId, Long networkId, String networkName) { + @Override + public Long reserveTier1PlacementAndGetPublicVlanId(long zoneId, long accountId, long domainId, Long vpcId, + Long networkId) { + if ((vpcId == null) == (networkId == null)) { + throw new IllegalArgumentException("Exactly one of vpcId or networkId is required"); + } + long resourceId = vpcId != null ? vpcId : networkId; + return nsxVrfGatewayLockManager.withPlacementLock(vpcId != null, resourceId, + () -> nsxVrfGatewayLockManager.withZoneLock(zoneId, () -> { + PlacementPlan plan = reservePlacement(zoneId, accountId, domainId, vpcId, networkId); + return plan.gateway == null ? null : plan.gateway.getPublicVlanDbId(); + })); + } + + @Override + public Long getPublicVlanId(long zoneId, long accountId, long domainId, Long vpcId, Long networkId) { + if ((vpcId == null) == (networkId == null)) { + throw new IllegalArgumentException("Exactly one of vpcId or networkId is required"); + } + long resourceId = vpcId != null ? vpcId : networkId; + return nsxVrfGatewayLockManager.withPlacementLock(vpcId != null, resourceId, () -> { + PlacementPlan plan = getRecordedPlacement(zoneId, accountId, domainId, vpcId, networkId); + return plan.gateway == null ? null : plan.gateway.getPublicVlanDbId(); + }); + } + + private PlacementPlan getRecordedPlacement(long zoneId, long accountId, long domainId, Long vpcId, + Long networkId) { + NsxVrfGatewayPlacementVO placement = vpcId != null + ? nsxVrfGatewayPlacementDao.findByVpcId(vpcId) + : nsxVrfGatewayPlacementDao.findByNetworkId(networkId); + if (placement == null) { + throw new CloudRuntimeException( + "The NSX Tier-1 has no recorded placement; backfill its current Tier-0 placement before continuing"); + } + validatePlacementOwner(placement, zoneId, accountId, domainId); + if (NsxVrfGatewayPlacementVO.State.FAILED.name().equals(placement.getState())) { + throw new CloudRuntimeException("The NSX Tier-1 placement is in a failed state"); + } + if (NsxVrfGatewayPlacementVO.State.PENDING_DELETE.name().equals(placement.getState())) { + throw new CloudRuntimeException("The NSX Tier-1 placement is pending deletion"); + } + NsxVrfGatewayVO gateway = placement.getGatewayId() == null + ? null : nsxVrfGatewayDao.findById(placement.getGatewayId()); + if (placement.getGatewayId() != null && gateway == null) { + throw new CloudRuntimeException("The NSX VRF gateway recorded for this Tier-1 placement is unavailable"); + } + return new PlacementPlan(gateway, placement); + } + + @Override + public void validatePublicIpVlan(long zoneId, long accountId, long domainId, Long vpcId, Long networkId, + long vlanId) { + Long expectedVlanId = getPublicVlanId(zoneId, accountId, domainId, vpcId, networkId); + if (expectedVlanId != null && expectedVlanId != vlanId) { + throw new CloudRuntimeException(String.format( + "Public IP range %s does not belong to the NSX VRF gateway used by this Tier-1", vlanId)); + } + } + + @Override + public boolean createNetwork(Long zoneId, long accountId, long domainId, Long networkId, String networkName, + boolean sourceNatEnabled) { CreateNsxTier1GatewayCommand createNsxTier1GatewayCommand = - new CreateNsxTier1GatewayCommand(domainId, accountId, zoneId, networkId, networkName, false, false); - NsxAnswer result = nsxControllerUtils.sendNsxCommand(createNsxTier1GatewayCommand, zoneId); - return result.getResult(); + new CreateNsxTier1GatewayCommand(domainId, accountId, zoneId, networkId, networkName, false, sourceNatEnabled); + return createTier1GatewayWithPlacement(createNsxTier1GatewayCommand, null); } public boolean deleteVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName) { DeleteNsxTier1GatewayCommand deleteNsxTier1GatewayCommand = new DeleteNsxTier1GatewayCommand(domainId, accountId, zoneId, vpcId, vpcName, true); - NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteNsxTier1GatewayCommand, zoneId); - return result.getResult(); + return deleteTier1GatewayWithPlacement(deleteNsxTier1GatewayCommand, vpcId, null); } public boolean deleteNetwork(long zoneId, long accountId, long domainId, NetworkVO network) { @@ -119,7 +538,7 @@ public boolean deleteNetwork(long zoneId, long accountId, long domainId, Network if (Objects.isNull(network.getVpcId())) { DeleteNsxTier1GatewayCommand deleteNsxTier1GatewayCommand = new DeleteNsxTier1GatewayCommand(domainId, accountId, zoneId, network.getId(), network.getName(), false); - result = nsxControllerUtils.sendNsxCommand(deleteNsxTier1GatewayCommand, zoneId); + return deleteTier1GatewayWithPlacement(deleteNsxTier1GatewayCommand, null, network.getId()); } return result.getResult(); } @@ -196,6 +615,171 @@ public boolean deleteFirewallRules(Network network, List netRule return result.getResult(); } + public NsxVpnGatewayResult createVpnGateway(Vpc vpc, String localEndpointIp) { + CreateNsxVpnGatewayCommand createNsxVpnGatewayCommand = new CreateNsxVpnGatewayCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), localEndpointIp); + NsxAnswer result = nsxControllerUtils.sendNsxCommandForResult(createNsxVpnGatewayCommand, vpc.getZoneId()); + return new NsxVpnGatewayResult(result.getResult(), result.isEndpointMayBeInUse()); + } + + public boolean deleteVpnGateway(Vpc vpc) { + DeleteNsxVpnGatewayCommand deleteNsxVpnGatewayCommand = new DeleteNsxVpnGatewayCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName()); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteNsxVpnGatewayCommand, vpc.getZoneId()); + return result.getResult(); + } + + public boolean createVpnConnection(Vpc vpc, String connectionUuid, String peerAddress, String psk, + String ikePolicy, String espPolicy, Long ikeLifetime, Long espLifetime, + boolean dpdEnabled, String ikeVersion, boolean passive, List peerCidrs, + String vtiLocalIp, String vtiPeerIp, int vtiPrefixLength, String localEndpointIp) { + CreateNsxVpnConnectionCommand createNsxVpnConnectionCommand = new CreateNsxVpnConnectionCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), connectionUuid, peerAddress, psk, + ikePolicy, espPolicy, ikeLifetime, espLifetime, dpdEnabled, ikeVersion, passive, peerCidrs, + vtiLocalIp, vtiPeerIp, vtiPrefixLength, vpc.getCidr(), localEndpointIp); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(createNsxVpnConnectionCommand, vpc.getZoneId()); + return result.getResult(); + } + + public boolean deleteVpnConnection(Vpc vpc, String connectionUuid) { + DeleteNsxVpnConnectionCommand deleteNsxVpnConnectionCommand = new DeleteNsxVpnConnectionCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), connectionUuid); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(deleteNsxVpnConnectionCommand, vpc.getZoneId()); + return result.getResult(); + } + + public boolean updateVpnConnectionState(Vpc vpc, String connectionUuid, boolean enabled) { + UpdateNsxVpnConnectionStateCommand command = new UpdateNsxVpnConnectionStateCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), connectionUuid, enabled); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, vpc.getZoneId()); + return result.getResult(); + } + + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + GetNsxVpnSessionStatusCommand getNsxVpnSessionStatusCommand = new GetNsxVpnSessionStatusCommand(vpc.getDomainId(), + vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), connectionUuid); + NsxAnswer result = nsxControllerUtils.sendNsxCommand(getNsxVpnSessionStatusCommand, vpc.getZoneId()); + return result.getDetails(); + } + + /** + * Every management server runs this poller over VPN connections whose gateway has persisted + * NSX ownership; duplicate polling in a multi-server setup is tolerated, as state transitions + * are serialized by the row lock in transitionVpnConnectionState + */ + protected class VpnStatusPollTask extends ManagedContextRunnable { + @Override + protected void runInContext() { + try { + Set polledConnectionIds = new HashSet<>(); + List connections = site2SiteVpnConnectionDao.listByStates( + VPN_POLLED_STATES.toArray(new Site2SiteVpnConnection.State[0])); + for (Site2SiteVpnConnectionVO connection : connections) { + Site2SiteVpnGatewayVO vpnGateway = site2SiteVpnGatewayDao.findById(connection.getVpnGatewayId()); + if (vpnGateway == null) { + continue; + } + VpcVO vpc = vpcDao.findById(vpnGateway.getVpcId()); + if (vpc == null || !isVpnProvidedByNsx(vpc, vpnGateway)) { + continue; + } + polledConnectionIds.add(connection.getId()); + pollVpnConnectionStatus(connection, vpc); + } + // Drop the failure counters of connections that were deleted or left the polled states + vpnStatusPollFailures.keySet().retainAll(polledConnectionIds); + } catch (Exception e) { + logger.warn("Failed to poll the status of the NSX Site-to-Site VPN connections: {}", e.getMessage(), e); + } + } + } + + private boolean isVpnProvidedByNsx(Vpc vpc, Site2SiteVpnGatewayVO vpnGateway) { + return vpc != null + && userIpAddressDetailsDao != null + && vpnGateway != null + && userIpAddressDetailsDao.findDetail(vpnGateway.getAddrId(), NsxElement.NSX_VPN_GATEWAY_IP_DETAIL) != null; + } + + protected void pollVpnConnectionStatus(Site2SiteVpnConnectionVO connection, VpcVO vpc) { + Site2SiteVpnConnection.State observedState = connection.getState(); + String status; + try { + status = getVpnConnectionStatus(vpc, connection.getUuid()); + vpnStatusPollFailures.remove(connection.getId()); + } catch (Exception e) { + int failures = vpnStatusPollFailures.merge(connection.getId(), 1, Integer::sum); + logger.warn("Failed to get the status of the NSX VPN connection {} of VPC {} ({} consecutive failure(s)): {}", + connection, vpc, failures, e.getMessage()); + if (failures >= VPN_STATUS_POLL_FAILURE_THRESHOLD) { + // A failed status query says nothing about the tunnel itself: alert, but never + // transition the connection state on management-plane errors + String title = String.format("Unable to poll the status of Site-to-site Vpn Connection %s", connection.getUuid()); + String context = String.format( + "The status of Site-to-site Vpn Connection %s on the NSX tier-1 gateway of VPC %s could not be polled %d consecutive times; its state %s is left unchanged", + connection.getUuid(), vpc.getName(), failures, observedState); + logger.warn(context); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, vpc.getZoneId(), null, title, context); + vpnStatusPollFailures.remove(connection.getId()); + } + return; + } + Site2SiteVpnConnection.State newState; + if (VPN_SESSION_STATUS_UP.equals(status)) { + newState = Site2SiteVpnConnection.State.Connected; + } else if (VPN_SESSION_STATUS_DOWN.equals(status) || VPN_SESSION_STATUS_DEGRADED.equals(status)) { + newState = Site2SiteVpnConnection.State.Disconnected; + } else if (VPN_SESSION_STATUS_NOT_FOUND.equals(status)) { + if (observedState == Site2SiteVpnConnection.State.Pending + || observedState == Site2SiteVpnConnection.State.Connecting) { + // the async connection job may still be creating the session on NSX + return; + } + if (observedState == Site2SiteVpnConnection.State.Disconnected) { + // stop intentionally disables the session; a subsequent status lookup may report it + // as absent while the connection remains a valid, stopped CloudStack resource + return; + } + // an established session vanished from NSX: flag the connection for a manual reset + newState = Site2SiteVpnConnection.State.Error; + } else { + logger.debug("NSX VPN connection {} of VPC {} reported the status {}, not transitioning the state", connection, vpc, status); + return; + } + transitionVpnConnectionState(connection, vpc, observedState, newState); + } + + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + if (observedState == newState) { + return; + } + Site2SiteVpnConnectionVO lock = site2SiteVpnConnectionDao.acquireInLockTable(connection.getId()); + if (lock == null) { + logger.warn("Unable to acquire the lock for the NSX Site-to-Site VPN connection {}, not updating its state", connection); + return; + } + try { + Site2SiteVpnConnectionVO lockedConnection = site2SiteVpnConnectionDao.findById(connection.getId()); + if (lockedConnection == null || !VPN_POLLED_STATES.contains(lockedConnection.getState()) + || lockedConnection.getState() != observedState) { + return; + } + Site2SiteVpnConnection.State oldState = lockedConnection.getState(); + lockedConnection.setState(newState); + site2SiteVpnConnectionDao.persist(lockedConnection); + vpnStatusPollFailures.remove(lockedConnection.getId()); + String title = String.format("Site-to-site Vpn Connection %s just switched from %s to %s", lockedConnection.getUuid(), oldState, newState); + String context = String.format("Site-to-site Vpn Connection %s on the NSX tier-1 gateway of VPC %s just switched from %s to %s", + lockedConnection.getUuid(), vpc.getName(), oldState, newState); + logger.info(context); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER, vpc.getZoneId(), null, title, context); + } finally { + site2SiteVpnConnectionDao.releaseFromLockTable(lock.getId()); + } + } + @Override public String getConfigComponentName() { return NsxApiClient.class.getSimpleName(); @@ -204,7 +788,8 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { - NSX_API_FAILURE_RETRIES, NSX_API_FAILURE_INTERVAL + NSX_API_FAILURE_RETRIES, NSX_API_FAILURE_INTERVAL, NSX_VPN_STATUS_POLL_INTERVAL, + NSX_VRF_SCOPE, NSX_VRF_FALLBACK_TO_SHARED_TIER0 }; } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxVrfGatewayLockManager.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxVrfGatewayLockManager.java new file mode 100644 index 000000000000..ba48177a679b --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxVrfGatewayLockManager.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.service; + +import java.util.function.Supplier; + +import org.springframework.stereotype.Component; + +import com.cloud.network.nsx.NsxService; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; + +@Component +public class NsxVrfGatewayLockManager { + + private static final int LOCK_TIMEOUT_SECONDS = 30; + + public T withZoneLock(long zoneId, Supplier operation) { + return withLock(NsxService.getVrfZoneLockName(zoneId), operation); + } + + public T withPlacementLock(boolean vpc, long resourceId, Supplier operation) { + return withLock(String.format("NsxVrfGateway.Placement.%s.%s", vpc ? "Vpc" : "Network", resourceId), + operation); + } + + private T withLock(String name, Supplier operation) { + GlobalLock lock = GlobalLock.getInternLock(name); + try { + if (!lock.lock(LOCK_TIMEOUT_SECONDS)) { + throw new CloudRuntimeException(String.format("Timed out waiting for lock %s", name)); + } + try { + return operation.get(); + } finally { + lock.unlock(); + } + } finally { + lock.releaseRef(); + } + } +} diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java index f44364f34c8b..278ac1fa4f98 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java @@ -51,6 +51,15 @@ public static String getNsxDistributedFirewallPolicyRuleId(String segmentName, l } public NsxAnswer sendNsxCommand(NsxCommand cmd, long zoneId) throws IllegalArgumentException { + NsxAnswer answer = sendNsxCommandForResult(cmd, zoneId); + if (!answer.getResult()) { + logger.error("NSX API Command failed"); + throw new InvalidParameterValueException("Failed API call to NSX controller"); + } + return answer; + } + + public NsxAnswer sendNsxCommandForResult(NsxCommand cmd, long zoneId) throws IllegalArgumentException { NsxProviderVO nsxProviderVO = nsxProviderDao.findByZoneId(zoneId); if (nsxProviderVO == null) { logger.error("No NSX controller was found!"); @@ -58,7 +67,7 @@ public NsxAnswer sendNsxCommand(NsxCommand cmd, long zoneId) throws IllegalArgum } Answer answer = agentMgr.easySend(nsxProviderVO.getHostId(), cmd); - if (answer == null || !answer.getResult()) { + if (answer == null) { logger.error("NSX API Command failed"); throw new InvalidParameterValueException("Failed API call to NSX controller"); } @@ -138,6 +147,50 @@ public static String getServerPoolMemberName(String tier1GatewayName, long vmId) return tier1GatewayName + "-VM" + vmId; } + public static String getVpnServiceName(String tier1GatewayName) { + return tier1GatewayName + "-vpn"; + } + + public static String getVpnLocalEndpointName(String vpnServiceName) { + return vpnServiceName + "-le"; + } + + public static String getVpnLocalEndpointNoSnatRuleName(String vpnServiceName) { + return vpnServiceName + "-le-nosnat"; + } + + public static String getVpnSessionName(String connectionUuid) { + return "cs-conn-" + connectionUuid; + } + + public static String getVpnIkeProfileName(String connectionUuid) { + return getVpnSessionName(connectionUuid) + "-ike"; + } + + public static String getVpnEspProfileName(String connectionUuid) { + return getVpnSessionName(connectionUuid) + "-esp"; + } + + public static String getVpnDpdProfileName(String connectionUuid) { + return getVpnSessionName(connectionUuid) + "-dpd"; + } + + public static String getVpnStaticRouteNamePrefix(String connectionUuid) { + return getVpnSessionName(connectionUuid) + "-route"; + } + + public static String getVpnStaticRouteName(String connectionUuid, int peerCidrIndex) { + return getVpnStaticRouteNamePrefix(connectionUuid) + peerCidrIndex; + } + + public static String getVpnNoSnatRuleNamePrefix(String connectionUuid) { + return getVpnSessionName(connectionUuid) + "-nosnat"; + } + + public static String getVpnNoSnatRuleName(String connectionUuid, int peerCidrIndex) { + return getVpnNoSnatRuleNamePrefix(connectionUuid) + peerCidrIndex; + } + public static String getLoadBalancerAlgorithm(String algorithm) { switch (algorithm) { case "leastconn": diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxHelper.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxHelper.java index b0668a0704f9..78c031fd335b 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxHelper.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxHelper.java @@ -22,6 +22,8 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcVO; import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.net.NetUtils; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; @@ -30,9 +32,27 @@ public class NsxHelper { + public static final int VPN_VTI_PREFIX_LENGTH = 30; + + private static final long VPN_VTI_SUBNET_BASE = NetUtils.ip2Long("169.254.64.0"); + // 169.254.64.0/18 provides 4096 /30 slots + private static final long VPN_VTI_SUBNET_SLOTS = 4096L; + private NsxHelper() { } + /** + * Derives the preferred VTI /30 for a Site-to-Site VPN connection from its database id: + * 169.254.64.0/18 base + 4 x (id mod 4096), local = .1 and peer = .2 within the /30. + * A collision is rejected rather than silently selecting another subnet: the peer must be + * configured with this deterministic pair, and CloudStack has no API field in which to persist + * an alternative allocation. + */ + public static Pair getVpnVtiAddressPair(long connectionId) { + long slotBase = VPN_VTI_SUBNET_BASE + (connectionId % VPN_VTI_SUBNET_SLOTS) * 4; + return new Pair<>(NetUtils.long2Ip(slotBase + 1), NetUtils.long2Ip(slotBase + 2)); + } + public static CreateNsxDhcpRelayConfigCommand createNsxDhcpRelayConfigCommand(DomainVO domain, Account account, DataCenter zone, VpcVO vpc, Network network, List addresses) { Long vpcId = vpc != null ? vpc.getId() : null; String vpcName = vpc != null ? vpc.getName() : null; @@ -41,8 +61,14 @@ public static CreateNsxDhcpRelayConfigCommand createNsxDhcpRelayConfigCommand(Do } public static CreateNsxSegmentCommand createNsxSegmentCommand(DomainVO domain, Account account, DataCenter zone, String vpcName, NetworkVO networkVO) { + return createNsxSegmentCommand(domain, account, zone, vpcName, networkVO, null, null, null); + } + + public static CreateNsxSegmentCommand createNsxSegmentCommand(DomainVO domain, Account account, DataCenter zone, String vpcName, NetworkVO networkVO, + String ipDiscoveryProfileId, String macDiscoveryProfileId, String segmentSecurityProfileId) { return new CreateNsxSegmentCommand(domain.getId(), account.getId(), zone.getId(), - networkVO.getVpcId(), vpcName, networkVO.getId(), networkVO.getName(), networkVO.getGateway(), networkVO.getCidr()); + networkVO.getVpcId(), vpcName, networkVO.getId(), networkVO.getName(), networkVO.getGateway(), networkVO.getCidr(), + ipDiscoveryProfileId, macDiscoveryProfileId, segmentSecurityProfileId); } public static CreateOrUpdateNsxTier1NatRuleCommand createOrUpdateNsxNatRuleCommand(long domainId, long accountId, long zoneId, diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxVpnCryptoUtils.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxVpnCryptoUtils.java new file mode 100644 index 000000000000..e4224ac934ed --- /dev/null +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxVpnCryptoUtils.java @@ -0,0 +1,181 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.exception.InvalidParameterValueException; + +/** + * Maps CloudStack Site-to-Site VPN policy strings (a comma-separated list of + * "cipher-hash[;dhgroup]" proposals) to the NSX IPSec VPN profile enums, rejecting + * parameters NSX does not support. + */ +public class NsxVpnCryptoUtils { + + public static final long NSX_MIN_IKE_SA_LIFETIME = 21600L; + public static final long NSX_MIN_ESP_SA_LIFETIME = 900L; + public static final long NSX_MAX_ESP_SA_LIFETIME = 31536000L; + public static final int NSX_MAX_PSK_LENGTH = 128; + + private static final Map ENCRYPTION_ALGORITHM_MAP = Map.of( + "aes128", "AES_128", + "aes256", "AES_256"); + + private static final Map DIGEST_ALGORITHM_MAP = Map.of( + "sha1", "SHA1", + "sha256", "SHA2_256", + "sha384", "SHA2_384", + "sha512", "SHA2_512"); + + private static final Map DH_GROUP_MAP = Map.of( + "modp1024", "GROUP2", + "modp1536", "GROUP5", + "modp2048", "GROUP14", + "modp3072", "GROUP15", + "modp4096", "GROUP16"); + + private static final Map IKE_VERSION_MAP = Map.of( + "ike", "IKE_FLEX", + "ikev1", "IKE_V1", + "ikev2", "IKE_V2"); + + private NsxVpnCryptoUtils() { + } + + public static List getEncryptionAlgorithms(String policy) { + Set algorithms = new LinkedHashSet<>(); + for (String entry : getPolicyEntries(policy)) { + String token = entry.split(";")[0].split("-")[0].toLowerCase(Locale.ROOT); + String nsxAlgorithm = ENCRYPTION_ALGORITHM_MAP.get(token); + if (nsxAlgorithm == null) { + throw new InvalidParameterValueException(String.format( + "Encryption algorithm %s is not supported by NSX Site-to-Site VPN (supported: %s)", + token, "aes128, aes256")); + } + algorithms.add(nsxAlgorithm); + } + return new ArrayList<>(algorithms); + } + + public static List getDigestAlgorithms(String policy) { + Set algorithms = new LinkedHashSet<>(); + for (String entry : getPolicyEntries(policy)) { + String[] tokens = entry.split(";")[0].split("-"); + if (tokens.length < 2) { + throw new InvalidParameterValueException(String.format( + "Missing hash algorithm in VPN policy entry %s", entry)); + } + String token = tokens[1].toLowerCase(Locale.ROOT); + String nsxAlgorithm = DIGEST_ALGORITHM_MAP.get(token); + if (nsxAlgorithm == null) { + throw new InvalidParameterValueException(String.format( + "Hash algorithm %s is not supported by NSX Site-to-Site VPN (supported: %s)", + token, "sha1, sha256, sha384, sha512")); + } + algorithms.add(nsxAlgorithm); + } + return new ArrayList<>(algorithms); + } + + public static List getDhGroups(String policy) { + Set groups = new LinkedHashSet<>(); + for (String entry : getPolicyEntries(policy)) { + String[] tokens = entry.split(";"); + if (tokens.length < 2 || StringUtils.isBlank(tokens[1])) { + continue; + } + String dhGroup = tokens[1].trim().toLowerCase(Locale.ROOT); + String nsxGroup = DH_GROUP_MAP.get(dhGroup); + if (nsxGroup == null) { + throw new InvalidParameterValueException(String.format( + "Diffie-Hellman group %s is not supported by NSX Site-to-Site VPN (supported: %s)", + dhGroup, "modp1024, modp1536, modp2048, modp3072, modp4096")); + } + groups.add(nsxGroup); + } + return new ArrayList<>(groups); + } + + public static String getIkeVersion(String ikeVersion) { + String token = StringUtils.isBlank(ikeVersion) ? "ike" : ikeVersion.toLowerCase(Locale.ROOT); + String nsxIkeVersion = IKE_VERSION_MAP.get(token); + if (nsxIkeVersion == null) { + throw new InvalidParameterValueException(String.format( + "IKE version %s is not supported by NSX Site-to-Site VPN (supported: %s)", + token, "ike, ikev1, ikev2")); + } + return nsxIkeVersion; + } + + public static void validateIkeLifetime(Long ikeLifetime) { + if (ikeLifetime != null && ikeLifetime < NSX_MIN_IKE_SA_LIFETIME) { + throw new InvalidParameterValueException(String.format( + "IKE lifetime %s is below the NSX minimum of %s seconds", ikeLifetime, NSX_MIN_IKE_SA_LIFETIME)); + } + } + + public static void validateEspLifetime(Long espLifetime) { + if (espLifetime != null && (espLifetime < NSX_MIN_ESP_SA_LIFETIME || espLifetime > NSX_MAX_ESP_SA_LIFETIME)) { + throw new InvalidParameterValueException(String.format( + "ESP lifetime %s is outside the NSX supported range of %s-%s seconds", + espLifetime, NSX_MIN_ESP_SA_LIFETIME, NSX_MAX_ESP_SA_LIFETIME)); + } + } + + public static void validatePresharedKey(String psk) { + if (StringUtils.isBlank(psk)) { + throw new InvalidParameterValueException("A pre-shared key is required for NSX Site-to-Site VPN"); + } + if (psk.length() > NSX_MAX_PSK_LENGTH) { + throw new InvalidParameterValueException(String.format( + "The pre-shared key exceeds the NSX maximum length of %s characters", NSX_MAX_PSK_LENGTH)); + } + } + + public static void validate(String ikePolicy, String espPolicy, String ikeVersion, + Long ikeLifetime, Long espLifetime, String psk) { + getEncryptionAlgorithms(ikePolicy); + getDigestAlgorithms(ikePolicy); + getDhGroups(ikePolicy); + getEncryptionAlgorithms(espPolicy); + getDigestAlgorithms(espPolicy); + getDhGroups(espPolicy); + getIkeVersion(ikeVersion); + validateIkeLifetime(ikeLifetime); + validateEspLifetime(espLifetime); + validatePresharedKey(psk); + } + + private static String[] getPolicyEntries(String policy) { + if (StringUtils.isBlank(policy)) { + throw new InvalidParameterValueException("An empty VPN policy cannot be mapped to NSX"); + } + String[] entries = policy.split(","); + for (int i = 0; i < entries.length; i++) { + entries[i] = entries[i].trim(); + } + return entries; + } +} diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java index 0d74bb8a3b3d..f74bb49a338e 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.resource; +import com.cloud.agent.api.Command; +import com.cloud.serializer.GsonHelper; import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.utils.exception.CloudRuntimeException; @@ -32,11 +34,17 @@ import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnGatewayCommand; import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnConnectionCommand; +import org.apache.cloudstack.agent.api.DeleteNsxVpnGatewayCommand; +import org.apache.cloudstack.agent.api.GetNsxVpnSessionStatusCommand; import org.apache.cloudstack.agent.api.NsxCommand; +import org.apache.cloudstack.agent.api.UpdateNsxVpnConnectionStateCommand; import org.apache.cloudstack.service.NsxApiClient; import org.apache.cloudstack.utils.NsxControllerUtils; import org.junit.After; @@ -44,6 +52,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -53,15 +62,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -184,6 +199,24 @@ public void testCreateNsxSegment() { assertTrue(answer.getResult()); } + @Test + public void testCreateNsxSegmentPassesProfileIdsToApiClient() { + List transportZoneList = List.of(new TransportZone.Builder().setDisplayName(transportZone).build()); + CreateNsxSegmentCommand command = new CreateNsxSegmentCommand(domainId, accountId, zoneId, + 2L, "VPC01", 3L, "Web", "10.10.10.1", "10.10.10.0/24", + "ip-profile", "mac-profile", "security-profile"); + when(nsxApi.getDefaultSiteId()).thenReturn("site1"); + when(nsxApi.getDefaultEnforcementPointPath("site1")).thenReturn("enforcementPointPath"); + when(nsxApi.getTransportZones()).thenReturn(transportZoneListResult); + when(transportZoneListResult.getResults()).thenReturn(transportZoneList); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertTrue(answer.getResult()); + verify(nsxApi).createSegment(anyString(), anyString(), anyString(), eq("enforcementPointPath"), + eq(transportZoneList), eq("ip-profile"), eq("mac-profile"), eq("security-profile")); + } + @Test public void testCreateNsxSegmentEmptySites() { when(nsxApi.getDefaultSiteId()).thenReturn(null); @@ -296,4 +329,173 @@ public void testCreateTier1NatRule() { NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); assertTrue(answer.getResult()); } + + @Test + public void testCreateNsxVpnGatewayRollsBackAfterFailure() { + CreateNsxVpnGatewayCommand command = new CreateNsxVpnGatewayCommand(domainId, accountId, zoneId, + 3L, "VPC01", "203.0.113.20"); + doThrow(new CloudRuntimeException("ERROR")).when(nsxApi).createVpnService(anyString(), anyString()); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + assertFalse(answer.isObjectExistent()); + assertFalse(answer.isEndpointMayBeInUse()); + verify(nsxApi).deleteVpnService("D1-A2-Z1-V3"); + } + + @Test + public void testCreateNsxVpnGatewayRefusesToAdoptExistingService() { + CreateNsxVpnGatewayCommand command = new CreateNsxVpnGatewayCommand(domainId, accountId, zoneId, + 3L, "VPC01", "203.0.113.20"); + when(nsxApi.isVpnServicePresent("D1-A2-Z1-V3")).thenReturn(true); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + assertTrue(answer.isObjectExistent()); + assertFalse(answer.isEndpointMayBeInUse()); + verify(nsxApi, never()).createVpnService(anyString(), anyString()); + verify(nsxApi, never()).deleteVpnService("D1-A2-Z1-V3"); + } + + @Test + public void testCreateNsxVpnGatewayReportsPossibleResourceWhenRollbackFails() { + CreateNsxVpnGatewayCommand command = new CreateNsxVpnGatewayCommand(domainId, accountId, zoneId, + 3L, "VPC01", "203.0.113.20"); + doThrow(new CloudRuntimeException("create failed")).when(nsxApi).createVpnService(anyString(), anyString()); + doThrow(new CloudRuntimeException("rollback failed")).when(nsxApi).deleteVpnService("D1-A2-Z1-V3"); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + assertTrue(answer.isEndpointMayBeInUse()); + } + + @Test + public void testCreateNsxVpnConnectionRollsBackNewSessionAfterRouteFailure() { + CreateNsxVpnConnectionCommand command = new CreateNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", "203.0.113.10", "psk", "aes256-sha256;modp2048", + "aes256-sha256;modp2048", 86400L, 3600L, true, "ikev2", false, + List.of("192.168.100.0/24"), "169.254.64.21", "169.254.64.22", 30, + "10.1.0.0/16", "203.0.113.20"); + when(nsxApi.getRouteBasedVpnSessionLocalVtiIps(anyString(), anyString())).thenReturn(Set.of()); + when(nsxApi.createRouteBasedVpnSession(anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyString(), anyInt())) + .thenReturn(NsxApiClient.VpnSessionProvisioningResult.CREATED); + doThrow(new CloudRuntimeException("ERROR")).when(nsxApi).addVpnConnectionRoutes(anyString(), anyString(), anyList(), anyString(), anyString()); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + verify(nsxApi).rollbackVpnConnection("D1-A2-Z1-V3", "connection-uuid"); + } + + @Test + public void testCreateNsxVpnConnectionDisablesPreexistingSessionAfterRouteFailure() { + CreateNsxVpnConnectionCommand command = new CreateNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", "203.0.113.10", "psk", "aes256-sha256;modp2048", + "aes256-sha256;modp2048", 86400L, 3600L, true, "ikev2", false, + List.of("192.168.100.0/24"), "169.254.64.21", "169.254.64.22", 30, + "10.1.0.0/16", "203.0.113.20"); + when(nsxApi.getRouteBasedVpnSessionLocalVtiIps(anyString(), anyString())).thenReturn(Set.of()); + when(nsxApi.createRouteBasedVpnSession(anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyString(), anyInt())) + .thenReturn(NsxApiClient.VpnSessionProvisioningResult.PREEXISTING); + doThrow(new CloudRuntimeException("ERROR")).when(nsxApi).addVpnConnectionRoutes(anyString(), anyString(), + anyList(), anyString(), anyString()); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + verify(nsxApi, never()).rollbackVpnConnection(anyString(), anyString()); + verify(nsxApi).updateVpnConnectionState("D1-A2-Z1-V3", "connection-uuid", false); + } + + @Test + public void testCreateNsxVpnConnectionEnablesSessionAfterRoutesAndNatExemptions() { + CreateNsxVpnConnectionCommand command = new CreateNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", "203.0.113.10", "psk", "aes256-sha256;modp2048", + "aes256-sha256;modp2048", 86400L, 3600L, true, "ikev2", false, + List.of("192.168.100.0/24"), "169.254.64.21", "169.254.64.22", 30, + "10.1.0.0/16", "203.0.113.20"); + when(nsxApi.getRouteBasedVpnSessionLocalVtiIps(anyString(), anyString())).thenReturn(Set.of()); + when(nsxApi.createRouteBasedVpnSession(anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyString(), anyInt())) + .thenReturn(NsxApiClient.VpnSessionProvisioningResult.CREATED); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertTrue(answer.getResult()); + InOrder inOrder = Mockito.inOrder(nsxApi); + inOrder.verify(nsxApi).createRouteBasedVpnSession(anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyString(), anyInt()); + inOrder.verify(nsxApi).addVpnConnectionRoutes("D1-A2-Z1-V3", "connection-uuid", + List.of("192.168.100.0/24"), "169.254.64.22", "10.1.0.0/16"); + inOrder.verify(nsxApi).ensureVpnNatExemptions("D1-A2-Z1-V3", "203.0.113.20"); + inOrder.verify(nsxApi).updateVpnConnectionState("D1-A2-Z1-V3", "connection-uuid", true); + } + + @Test + public void testCreateNsxVpnConnectionRejectsVtiCollisionBeforeCreate() { + CreateNsxVpnConnectionCommand command = new CreateNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", "203.0.113.10", "psk", "aes256-sha256;modp2048", + "aes256-sha256;modp2048", 86400L, 3600L, true, "ikev2", false, + List.of("192.168.100.0/24"), "169.254.64.21", "169.254.64.22", 30, + "10.1.0.0/16", "203.0.113.20"); + when(nsxApi.getRouteBasedVpnSessionLocalVtiIps(anyString(), anyString())).thenReturn(Set.of("169.254.64.21")); + + NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(command); + + assertFalse(answer.getResult()); + verify(nsxApi, Mockito.never()).createRouteBasedVpnSession(anyString(), anyString(), anyString(), anyString(), + anyString(), anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyString(), anyInt()); + } + + @Test + public void testNsxVpnLifecycleCommandsDispatch() { + DeleteNsxVpnGatewayCommand deleteGateway = new DeleteNsxVpnGatewayCommand(domainId, accountId, zoneId, 3L, "VPC01"); + DeleteNsxVpnConnectionCommand deleteConnection = new DeleteNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid"); + UpdateNsxVpnConnectionStateCommand update = new UpdateNsxVpnConnectionStateCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", false); + GetNsxVpnSessionStatusCommand status = new GetNsxVpnSessionStatusCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid"); + when(nsxApi.getVpnSessionStatus("D1-A2-Z1-V3", "connection-uuid")).thenReturn("UP"); + + assertTrue(((NsxAnswer) nsxResource.executeRequest(deleteGateway)).getResult()); + assertTrue(((NsxAnswer) nsxResource.executeRequest(deleteConnection)).getResult()); + assertTrue(((NsxAnswer) nsxResource.executeRequest(update)).getResult()); + NsxAnswer statusAnswer = (NsxAnswer) nsxResource.executeRequest(status); + assertTrue(statusAnswer.getResult()); + assertTrue(statusAnswer.getDetails().contains("UP")); + verify(nsxApi).deleteVpnService("D1-A2-Z1-V3"); + verify(nsxApi).deleteVpnConnection("D1-A2-Z1-V3", "connection-uuid"); + verify(nsxApi).updateVpnConnectionState("D1-A2-Z1-V3", "connection-uuid", false); + } + + @Test + public void testNsxVpnConnectionCommandRoundTripsThroughCloudStackWireAdaptor() { + CreateNsxVpnConnectionCommand command = new CreateNsxVpnConnectionCommand(domainId, accountId, zoneId, + 3L, "VPC01", "connection-uuid", "203.0.113.10", "secret-psk", "aes256-sha256;modp2048", + "aes256-sha256;modp2048", 86400L, 3600L, true, "ikev2", true, + List.of("192.168.100.0/24", "192.168.101.0/24"), "169.254.64.21", "169.254.64.22", 30, + "10.1.0.0/16", "203.0.113.20"); + + String wire = GsonHelper.getGson().toJson(new Command[]{command}, Command[].class); + String logPayload = GsonHelper.getGsonLogger().toJson(new Command[]{command}, Command[].class); + Command[] decoded = GsonHelper.getGson().fromJson(wire, Command[].class); + + assertTrue(wire.contains("secret-psk")); + assertFalse(logPayload.contains("secret-psk")); + assertFalse(logPayload.contains("\"psk\"")); + assertTrue(decoded[0] instanceof CreateNsxVpnConnectionCommand); + CreateNsxVpnConnectionCommand decodedCommand = (CreateNsxVpnConnectionCommand) decoded[0]; + assertEquals("secret-psk", decodedCommand.getPsk()); + assertEquals(command.getPeerCidrs(), decodedCommand.getPeerCidrs()); + assertEquals(command.getVtiLocalIp(), decodedCommand.getVtiLocalIp()); + assertEquals(command.getVtiPeerIp(), decodedCommand.getVtiPeerIp()); + assertEquals(command.getIkeLifetime(), decodedCommand.getIkeLifetime()); + assertEquals(command.getEspLifetime(), decodedCommand.getEspLifetime()); + } } diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxApiClientTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxApiClientTest.java index 5f8d771f75df..4c4f22c6df83 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxApiClientTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxApiClientTest.java @@ -22,14 +22,40 @@ import com.vmware.nsx.cluster.Status; import com.vmware.nsx.model.ClusterStatus; import com.vmware.nsx.model.ControllerClusterStatus; +import com.vmware.nsx_policy.Infra; +import com.vmware.nsx_policy.infra.IpDiscoveryProfiles; import com.vmware.nsx_policy.infra.LbAppProfiles; import com.vmware.nsx_policy.infra.LbMonitorProfiles; import com.vmware.nsx_policy.infra.LbPools; import com.vmware.nsx_policy.infra.LbServices; import com.vmware.nsx_policy.infra.LbVirtualServers; +import com.vmware.nsx_policy.infra.IpsecVpnDpdProfiles; +import com.vmware.nsx_policy.infra.IpsecVpnIkeProfiles; +import com.vmware.nsx_policy.infra.IpsecVpnTunnelProfiles; +import com.vmware.nsx_policy.infra.MacDiscoveryProfiles; +import com.vmware.nsx_policy.infra.SegmentSecurityProfiles; +import com.vmware.nsx_policy.infra.Segments; +import com.vmware.nsx_policy.infra.Tier1s; +import com.vmware.nsx_policy.infra.Tier0s; +import com.vmware.nsx_policy.infra.tier_0s.locale_services.Interfaces; +import com.vmware.nsx_policy.infra.tier_1s.IpsecVpnServices; +import com.vmware.nsx_policy.infra.tier_1s.LocaleServices; +import com.vmware.nsx_policy.infra.tier_1s.StaticRoutes; +import com.vmware.nsx_policy.infra.tier_1s.nat.NatRules; import com.vmware.nsx_policy.infra.domains.Groups; +import com.vmware.nsx_policy.infra.tier_1s.ipsec_vpn_services.Sessions; import com.vmware.nsx_policy.model.ApiError; +import com.vmware.nsx_policy.model.ChildSegment; +import com.vmware.nsx_policy.model.ChildSegmentDiscoveryProfileBindingMap; +import com.vmware.nsx_policy.model.ChildSegmentSecurityProfileBindingMap; import com.vmware.nsx_policy.model.Group; +import com.vmware.nsx_policy.model.IPDiscoveryProfile; +import com.vmware.nsx_policy.model.IPSecVpnDpdProfile; +import com.vmware.nsx_policy.model.IPSecVpnIkeProfile; +import com.vmware.nsx_policy.model.IPSecVpnSession; +import com.vmware.nsx_policy.model.IPSecVpnServiceListResult; +import com.vmware.nsx_policy.model.IPSecVpnTunnelInterface; +import com.vmware.nsx_policy.model.IPSecVpnTunnelProfile; import com.vmware.nsx_policy.model.LBAppProfileListResult; import com.vmware.nsx_policy.model.LBIcmpMonitorProfile; import com.vmware.nsx_policy.model.LBService; @@ -37,7 +63,25 @@ import com.vmware.nsx_policy.model.LBPool; import com.vmware.nsx_policy.model.LBPoolMember; import com.vmware.nsx_policy.model.LBVirtualServer; +import com.vmware.nsx_policy.model.LocaleServicesListResult; +import com.vmware.nsx_policy.model.MacDiscoveryProfile; import com.vmware.nsx_policy.model.PathExpression; +import com.vmware.nsx_policy.model.PolicyNatRule; +import com.vmware.nsx_policy.model.PolicyNatRuleListResult; +import com.vmware.nsx_policy.model.PolicyBgpNeighborStatus; +import com.vmware.nsx_policy.model.PolicyBgpNeighborsStatusListResult; +import com.vmware.nsx_policy.model.RouteBasedIPSecVpnSession; +import com.vmware.nsx_policy.model.Segment; +import com.vmware.nsx_policy.model.SegmentDiscoveryProfileBindingMap; +import com.vmware.nsx_policy.model.SegmentSecurityProfile; +import com.vmware.nsx_policy.model.StaticRoutesListResult; +import com.vmware.nsx_policy.model.Tag; +import com.vmware.nsx_policy.model.Tier1; +import com.vmware.nsx_policy.model.Tier0; +import com.vmware.nsx_policy.model.Tier0Interface; +import com.vmware.nsx_policy.model.Tier0InterfaceListResult; +import com.vmware.nsx_policy.model.Tier0VrfConfig; +import com.vmware.nsx_policy.model.TunnelInterfaceIPSubnet; import com.vmware.vapi.bindings.Service; import com.vmware.vapi.bindings.Structure; import com.vmware.vapi.std.errors.Error; @@ -47,20 +91,27 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.InOrder; import java.util.List; import java.util.function.Function; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -102,6 +153,187 @@ public void testCreateGroupForSegment() { } } + @Test + public void testCreateSegmentBindsConfiguredProfiles() { + Infra infraService = Mockito.mock(Infra.class); + IpDiscoveryProfiles ipProfiles = Mockito.mock(IpDiscoveryProfiles.class); + MacDiscoveryProfiles macProfiles = Mockito.mock(MacDiscoveryProfiles.class); + SegmentSecurityProfiles securityProfiles = Mockito.mock(SegmentSecurityProfiles.class); + IPDiscoveryProfile ipProfile = Mockito.mock(IPDiscoveryProfile.class); + MacDiscoveryProfile macProfile = Mockito.mock(MacDiscoveryProfile.class); + SegmentSecurityProfile securityProfile = Mockito.mock(SegmentSecurityProfile.class); + when(nsxService.apply(Infra.class)).thenReturn(infraService); + when(nsxService.apply(IpDiscoveryProfiles.class)).thenReturn(ipProfiles); + when(nsxService.apply(MacDiscoveryProfiles.class)).thenReturn(macProfiles); + when(nsxService.apply(SegmentSecurityProfiles.class)).thenReturn(securityProfiles); + when(ipProfiles.get("ip-profile")).thenReturn(ipProfile); + when(macProfiles.get("mac-profile")).thenReturn(macProfile); + when(securityProfiles.get("security-profile")).thenReturn(securityProfile); + when(ipProfile.getId()).thenReturn("ip-profile"); + when(macProfile.getId()).thenReturn("mac-profile"); + when(securityProfile.getId()).thenReturn("security-profile"); + when(ipProfile.getPath()).thenReturn("/infra/ip-discovery-profiles/ip-profile"); + when(macProfile.getPath()).thenReturn("/infra/mac-discovery-profiles/mac-profile"); + when(securityProfile.getPath()).thenReturn("/infra/segment-security-profiles/security-profile"); + ArgumentCaptor infraCaptor = + ArgumentCaptor.forClass(com.vmware.nsx_policy.model.Infra.class); + + client.createSegment("segment", "tier1", "10.10.10.1/24", "/infra/sites/default/enforcement-points/default", + List.of(new com.vmware.nsx.model.TransportZone.Builder().setId("tz").build()), + "ip-profile", "mac-profile", "security-profile"); + + verify(infraService).patch(infraCaptor.capture(), eq(false)); + verify(nsxService, never()).apply(Segments.class); + Assert.assertEquals("Infra", infraCaptor.getValue().getResourceType()); + Assert.assertEquals(1, infraCaptor.getValue().getChildren().size()); + ChildSegment childSegment = (ChildSegment) infraCaptor.getValue().getChildren().get(0); + Assert.assertEquals("ChildSegment", childSegment.getResourceType()); + Assert.assertEquals("segment", childSegment.getId()); + Assert.assertEquals("Segment", childSegment.getSegment().getResourceType()); + Assert.assertEquals("segment", childSegment.getSegment().getId()); + Assert.assertEquals(2, childSegment.getSegment().getChildren().size()); + ChildSegmentDiscoveryProfileBindingMap discoveryChild = (ChildSegmentDiscoveryProfileBindingMap) + childSegment.getSegment().getChildren().get(0); + Assert.assertEquals("ChildSegmentDiscoveryProfileBindingMap", discoveryChild.getResourceType()); + SegmentDiscoveryProfileBindingMap discoveryBinding = discoveryChild.getSegmentDiscoveryProfileBindingMap(); + Assert.assertEquals("cloudstack-discovery-profile-binding", discoveryChild.getId()); + Assert.assertEquals("cloudstack-discovery-profile-binding", discoveryBinding.getId()); + Assert.assertEquals("SegmentDiscoveryProfileBindingMap", discoveryBinding.getResourceType()); + Assert.assertEquals("/infra/ip-discovery-profiles/ip-profile", discoveryBinding.getIpDiscoveryProfilePath()); + Assert.assertEquals("/infra/mac-discovery-profiles/mac-profile", discoveryBinding.getMacDiscoveryProfilePath()); + ChildSegmentSecurityProfileBindingMap securityChild = (ChildSegmentSecurityProfileBindingMap) + childSegment.getSegment().getChildren().get(1); + Assert.assertEquals("cloudstack-security-profile-binding", securityChild.getId()); + Assert.assertEquals("cloudstack-security-profile-binding", + securityChild.getSegmentSecurityProfileBindingMap().getId()); + Assert.assertEquals("ChildSegmentSecurityProfileBindingMap", securityChild.getResourceType()); + Assert.assertEquals("SegmentSecurityProfileBindingMap", + securityChild.getSegmentSecurityProfileBindingMap().getResourceType()); + Assert.assertEquals("/infra/segment-security-profiles/security-profile", + securityChild.getSegmentSecurityProfileBindingMap().getSegmentSecurityProfilePath()); + } + + @Test + public void testSegmentProfileBindingsWithOnlyIpDiscoveryProfile() { + List bindings = client.getSegmentProfileBindings("/infra/ip-discovery-profiles/ip-profile", null, null); + + Assert.assertEquals(1, bindings.size()); + ChildSegmentDiscoveryProfileBindingMap child = (ChildSegmentDiscoveryProfileBindingMap) bindings.get(0); + Assert.assertEquals("ChildSegmentDiscoveryProfileBindingMap", child.getResourceType()); + Assert.assertEquals("SegmentDiscoveryProfileBindingMap", + child.getSegmentDiscoveryProfileBindingMap().getResourceType()); + Assert.assertEquals("/infra/ip-discovery-profiles/ip-profile", + child.getSegmentDiscoveryProfileBindingMap().getIpDiscoveryProfilePath()); + Assert.assertNull(child.getSegmentDiscoveryProfileBindingMap().getMacDiscoveryProfilePath()); + } + + @Test + public void testSegmentProfileBindingsWithOnlyMacDiscoveryProfile() { + List bindings = client.getSegmentProfileBindings(null, "/infra/mac-discovery-profiles/mac-profile", null); + + Assert.assertEquals(1, bindings.size()); + ChildSegmentDiscoveryProfileBindingMap child = (ChildSegmentDiscoveryProfileBindingMap) bindings.get(0); + Assert.assertEquals("ChildSegmentDiscoveryProfileBindingMap", child.getResourceType()); + Assert.assertEquals("SegmentDiscoveryProfileBindingMap", + child.getSegmentDiscoveryProfileBindingMap().getResourceType()); + Assert.assertNull(child.getSegmentDiscoveryProfileBindingMap().getIpDiscoveryProfilePath()); + Assert.assertEquals("/infra/mac-discovery-profiles/mac-profile", + child.getSegmentDiscoveryProfileBindingMap().getMacDiscoveryProfilePath()); + } + + @Test + public void testSegmentProfileBindingsWithOnlySecurityProfile() { + List bindings = client.getSegmentProfileBindings(null, null, + "/infra/segment-security-profiles/security-profile"); + + Assert.assertEquals(1, bindings.size()); + ChildSegmentSecurityProfileBindingMap child = (ChildSegmentSecurityProfileBindingMap) bindings.get(0); + Assert.assertEquals("ChildSegmentSecurityProfileBindingMap", child.getResourceType()); + Assert.assertEquals("SegmentSecurityProfileBindingMap", + child.getSegmentSecurityProfileBindingMap().getResourceType()); + Assert.assertEquals("/infra/segment-security-profiles/security-profile", + child.getSegmentSecurityProfileBindingMap().getSegmentSecurityProfilePath()); + } + + @Test + public void testCreateSegmentWithoutProfilesPreservesExistingBehavior() { + Segments segmentService = Mockito.mock(Segments.class); + when(nsxService.apply(Segments.class)).thenReturn(segmentService); + + client.createSegment("segment", "tier1", "10.10.10.1/24", "/infra/sites/default/enforcement-points/default", + List.of(new com.vmware.nsx.model.TransportZone.Builder().setId("tz").build())); + + verify(segmentService).patch(eq("segment"), any(Segment.class)); + verify(nsxService, never()).apply(IpDiscoveryProfiles.class); + verify(nsxService, never()).apply(MacDiscoveryProfiles.class); + verify(nsxService, never()).apply(SegmentSecurityProfiles.class); + verify(nsxService, never()).apply(Infra.class); + } + + @Test + public void testCreateSegmentRejectsProfileWithoutCanonicalPathBeforeCreatingSegment() { + Segments segmentService = Mockito.mock(Segments.class); + IpDiscoveryProfiles ipProfiles = Mockito.mock(IpDiscoveryProfiles.class); + IPDiscoveryProfile ipProfile = Mockito.mock(IPDiscoveryProfile.class); + when(nsxService.apply(Segments.class)).thenReturn(segmentService); + when(nsxService.apply(IpDiscoveryProfiles.class)).thenReturn(ipProfiles); + when(ipProfiles.get("ip-profile")).thenReturn(ipProfile); + when(ipProfile.getId()).thenReturn("ip-profile"); + when(ipProfile.getPath()).thenReturn(" "); + + Assert.assertThrows(CloudRuntimeException.class, + () -> client.createSegment("segment", "tier1", "10.10.10.1/24", + "/infra/sites/default/enforcement-points/default", + List.of(new com.vmware.nsx.model.TransportZone.Builder().setId("tz").build()), + "ip-profile", null, null)); + + verify(segmentService, never()).patch(anyString(), any(Segment.class)); + verify(nsxService, never()).apply(Infra.class); + } + + @Test + public void testCreateSegmentRejectsProfileMarkedForDeletionBeforeCreatingSegment() { + Segments segmentService = Mockito.mock(Segments.class); + IpDiscoveryProfiles ipProfiles = Mockito.mock(IpDiscoveryProfiles.class); + IPDiscoveryProfile ipProfile = Mockito.mock(IPDiscoveryProfile.class); + when(nsxService.apply(Segments.class)).thenReturn(segmentService); + when(nsxService.apply(IpDiscoveryProfiles.class)).thenReturn(ipProfiles); + when(ipProfiles.get("ip-profile")).thenReturn(ipProfile); + when(ipProfile.getId()).thenReturn("ip-profile"); + when(ipProfile.getPath()).thenReturn("/infra/ip-discovery-profiles/ip-profile"); + when(ipProfile.getMarkedForDelete()).thenReturn(true); + + Assert.assertThrows(CloudRuntimeException.class, + () -> client.createSegment("segment", "tier1", "10.10.10.1/24", + "/infra/sites/default/enforcement-points/default", + List.of(new com.vmware.nsx.model.TransportZone.Builder().setId("tz").build()), + "ip-profile", null, null)); + + verify(segmentService, never()).patch(anyString(), any(Segment.class)); + verify(nsxService, never()).apply(Infra.class); + } + + @Test + public void testCreateSegmentRejectsProfileResolvedWithDifferentIdBeforeCreatingSegment() { + Segments segmentService = Mockito.mock(Segments.class); + IpDiscoveryProfiles ipProfiles = Mockito.mock(IpDiscoveryProfiles.class); + IPDiscoveryProfile ipProfile = Mockito.mock(IPDiscoveryProfile.class); + when(nsxService.apply(Segments.class)).thenReturn(segmentService); + when(nsxService.apply(IpDiscoveryProfiles.class)).thenReturn(ipProfiles); + when(ipProfiles.get("ip-profile")).thenReturn(ipProfile); + when(ipProfile.getId()).thenReturn("different-profile"); + when(ipProfile.getPath()).thenReturn("/infra/ip-discovery-profiles/different-profile"); + + Assert.assertThrows(CloudRuntimeException.class, + () -> client.createSegment("segment", "tier1", "10.10.10.1/24", + "/infra/sites/default/enforcement-points/default", + List.of(new com.vmware.nsx.model.TransportZone.Builder().setId("tz").build()), + "ip-profile", null, null)); + + verify(segmentService, never()).patch(anyString(), any(Segment.class)); + verify(nsxService, never()).apply(Infra.class); + } + @Test public void testGetGroupsForTrafficIngress() { SDNProviderNetworkRule rule = Mockito.mock(SDNProviderNetworkRule.class); @@ -391,6 +623,682 @@ public void testCreateNsxLbServerPoolThrowsExceptionOnPatchError() { assertTrue(thrownException.getMessage().startsWith("Failed to create NSX LB server pool, due to")); } + @Test + public void testVpnNatOrderingRestoresOriginalSourceNatSequenceAndTags() { + NatRules natRules = Mockito.mock(NatRules.class); + PolicyNatRule sourceNatRule = new PolicyNatRule.Builder() + .setId("t1-NAT") + .setDisplayName("CloudStack source NAT") + .setAction("SNAT") + .setTranslatedNetwork("203.0.113.10") + .setSourceNetwork("0.0.0.0/0") + .setDestinationNetwork("ANY") + .setSequenceNumber(42L) + .setTags(List.of(new Tag.Builder().setScope("owner").setTag("cloudstack").build())) + .setEnabled(true) + .build(); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + Mockito.when(natRules.get(TIER_1_GATEWAY_NAME, "USER", "t1-NAT")).thenReturn(sourceNatRule); + + client.ensureVpnNatExemptions(TIER_1_GATEWAY_NAME, "203.0.113.20"); + + ArgumentCaptor demotedCaptor = ArgumentCaptor.forClass(PolicyNatRule.class); + verify(natRules).patch(eq(TIER_1_GATEWAY_NAME), eq("USER"), eq("t1-NAT"), demotedCaptor.capture()); + PolicyNatRule demotedRule = demotedCaptor.getValue(); + assertEquals(Long.valueOf(1000L), demotedRule.getSequenceNumber()); + assertEquals("203.0.113.10", demotedRule.getTranslatedNetwork()); + assertTrue(demotedRule.getTags().stream().anyMatch(tag -> "owner".equals(tag.getScope()) + && "cloudstack".equals(tag.getTag()))); + assertTrue(demotedRule.getTags().stream().anyMatch(tag -> "cloudstack-vpn-original-snat-sequence".equals(tag.getScope()) + && "42".equals(tag.getTag()))); + + clearInvocations(natRules); + Mockito.when(natRules.get(TIER_1_GATEWAY_NAME, "USER", "t1-NAT")).thenReturn(demotedRule); + client.restoreSourceNatRuleSequence(TIER_1_GATEWAY_NAME); + + ArgumentCaptor restoredCaptor = ArgumentCaptor.forClass(PolicyNatRule.class); + verify(natRules).patch(eq(TIER_1_GATEWAY_NAME), eq("USER"), eq("t1-NAT"), restoredCaptor.capture()); + PolicyNatRule restoredRule = restoredCaptor.getValue(); + assertEquals(Long.valueOf(42L), restoredRule.getSequenceNumber()); + assertEquals("203.0.113.10", restoredRule.getTranslatedNetwork()); + assertEquals(1, restoredRule.getTags().size()); + assertEquals("owner", restoredRule.getTags().get(0).getScope()); + assertEquals("cloudstack", restoredRule.getTags().get(0).getTag()); + } + + @Test + public void testCreateTier1GatewayReconcilesExistingGatewayAndLocaleService() { + Tier1s tier1s = Mockito.mock(Tier1s.class); + Tier1 existing = Mockito.mock(Tier1.class); + com.vmware.nsx_policy.infra.tier_0s.LocaleServices tier0LocaleServices = + Mockito.mock(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class); + LocaleServices tier1LocaleServices = Mockito.mock(LocaleServices.class); + LocaleServicesListResult localeServicesResult = Mockito.mock(LocaleServicesListResult.class); + com.vmware.nsx_policy.model.LocaleServices tier0Locale = + Mockito.mock(com.vmware.nsx_policy.model.LocaleServices.class); + when(nsxService.apply(Tier1s.class)).thenReturn(tier1s); + when(nsxService.apply(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class)) + .thenReturn(tier0LocaleServices); + when(nsxService.apply(LocaleServices.class)).thenReturn(tier1LocaleServices); + when(tier1s.get(TIER_1_GATEWAY_NAME)).thenReturn(existing); + when(existing.getTier0Path()).thenReturn("/infra/tier-0s/t0"); + when(tier0LocaleServices.list(eq("t0"), nullable(String.class), eq(false), nullable(String.class), + eq(1000L), nullable(Boolean.class), nullable(String.class))).thenReturn(localeServicesResult); + when(localeServicesResult.getResults()).thenReturn(List.of(tier0Locale)); + when(tier0Locale.getEdgeClusterPath()).thenReturn( + "/infra/sites/default/enforcement-points/default/edge-clusters/ec1"); + ArgumentCaptor tier1Captor = ArgumentCaptor.forClass(Tier1.class); + + client.createTier1Gateway(TIER_1_GATEWAY_NAME, "t0", "ec1", true); + + verify(tier1s).patch(eq(TIER_1_GATEWAY_NAME), tier1Captor.capture()); + assertEquals("/infra/tier-0s/t0", tier1Captor.getValue().getTier0Path()); + assertTrue(tier1Captor.getValue().getRouteAdvertisementTypes().contains("TIER1_NAT")); + verify(tier1LocaleServices).patch(eq(TIER_1_GATEWAY_NAME), eq("default"), + any(com.vmware.nsx_policy.model.LocaleServices.class)); + } + + @Test + public void testCreateTier1GatewayRejectsExistingGatewayOnDifferentParentWithoutMutation() { + Tier1s tier1s = Mockito.mock(Tier1s.class); + Tier1 existing = Mockito.mock(Tier1.class); + when(nsxService.apply(Tier1s.class)).thenReturn(tier1s); + when(tier1s.get(TIER_1_GATEWAY_NAME)).thenReturn(existing); + when(existing.getTier0Path()).thenReturn("/infra/tier-0s/other-t0"); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.createTier1Gateway(TIER_1_GATEWAY_NAME, "t0", "ec1", true)); + + assertTrue(exception.getMessage().contains("already exists under")); + verify(tier1s, never()).patch(anyString(), any(Tier1.class)); + verify(nsxService, never()).apply(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class); + } + + @Test + public void testValidateVrfGatewayReturnsCanonicalBackendPaths() { + Tier0s tier0s = Mockito.mock(Tier0s.class); + Tier0 tier0 = Mockito.mock(Tier0.class); + com.vmware.nsx_policy.infra.tier_0s.LocaleServices localeServices = + Mockito.mock(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class); + LocaleServicesListResult localeServicesResult = Mockito.mock(LocaleServicesListResult.class); + com.vmware.nsx_policy.model.LocaleServices localeService = + Mockito.mock(com.vmware.nsx_policy.model.LocaleServices.class); + Interfaces interfaces = Mockito.mock(Interfaces.class); + Tier0InterfaceListResult interfaceResult = Mockito.mock(Tier0InterfaceListResult.class); + com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status statusService = + Mockito.mock(com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status.class); + PolicyBgpNeighborsStatusListResult statuses = Mockito.mock(PolicyBgpNeighborsStatusListResult.class); + PolicyBgpNeighborStatus establishedStatus = Mockito.mock(PolicyBgpNeighborStatus.class); + Tier0VrfConfig vrfConfig = Mockito.mock(Tier0VrfConfig.class); + String parentPath = "/infra/tier-0s/parent-t0"; + String edgeClusterPath = "/infra/sites/default/enforcement-points/default/edge-clusters/ec1"; + + when(nsxService.apply(Tier0s.class)).thenReturn(tier0s); + when(nsxService.apply(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class)) + .thenReturn(localeServices); + when(nsxService.apply(Interfaces.class)).thenReturn(interfaces); + when(nsxService.apply(com.vmware.nsx_policy.infra.tier_0s.locale_services.bgp.neighbors.Status.class)) + .thenReturn(statusService); + when(tier0s.get("vrf-t0")).thenReturn(tier0); + when(tier0.getVrfConfig()).thenReturn(vrfConfig); + when(vrfConfig.getTier0Path()).thenReturn(parentPath); + when(localeServices.list(eq("vrf-t0"), nullable(String.class), eq(false), nullable(String.class), + eq(1000L), nullable(Boolean.class), nullable(String.class))).thenReturn(localeServicesResult); + when(localeServicesResult.getResults()).thenReturn(List.of(localeService)); + when(localeService.getId()).thenReturn("default"); + when(localeService.getEdgeClusterPath()).thenReturn(edgeClusterPath); + when(interfaces.list(eq("vrf-t0"), eq("default"), nullable(String.class), eq(false), + nullable(String.class), eq(1L), nullable(Boolean.class), nullable(String.class))) + .thenReturn(interfaceResult); + when(interfaceResult.getResults()).thenReturn(List.of(Mockito.mock(Tier0Interface.class))); + when(statusService.list(eq("vrf-t0"), eq("default"), nullable(String.class), nullable(String.class), + nullable(String.class), eq(false), nullable(String.class), nullable(Long.class), + nullable(Boolean.class), nullable(String.class))).thenReturn(statuses); + when(statuses.getResults()).thenReturn(List.of(establishedStatus)); + when(establishedStatus.getConnectionState()) + .thenReturn(PolicyBgpNeighborStatus.CONNECTION_STATE_ESTABLISHED); + + NsxApiClient.VrfGatewayValidation result = client.validateVrfGateway( + "vrf-t0", "parent-t0", "ec1"); + + assertEquals(parentPath, result.getParentTier0Path()); + assertEquals(edgeClusterPath, result.getEdgeClusterPath()); + } + + @Test + public void testValidateVrfGatewayRejectsParentMismatchBeforePlacementChecks() { + Tier0s tier0s = Mockito.mock(Tier0s.class); + Tier0 tier0 = Mockito.mock(Tier0.class); + Tier0VrfConfig vrfConfig = Mockito.mock(Tier0VrfConfig.class); + when(nsxService.apply(Tier0s.class)).thenReturn(tier0s); + when(tier0s.get("vrf-t0")).thenReturn(tier0); + when(tier0.getVrfConfig()).thenReturn(vrfConfig); + when(vrfConfig.getTier0Path()).thenReturn("/infra/tier-0s/other-parent"); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.validateVrfGateway("vrf-t0", "parent-t0", "ec1")); + + assertTrue(exception.getMessage().contains("has parent")); + verify(nsxService, never()).apply(com.vmware.nsx_policy.infra.tier_0s.LocaleServices.class); + } + + @Test + public void testDeleteTier1GatewayRemovesVpnResourcesBeforeLocaleServices() { + Tier1s tier1s = Mockito.mock(Tier1s.class); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + IpsecVpnServices vpnServices = Mockito.mock(IpsecVpnServices.class); + LocaleServices localeServices = Mockito.mock(LocaleServices.class); + StaticRoutesListResult staticRoutesResult = Mockito.mock(StaticRoutesListResult.class); + PolicyNatRuleListResult natRulesResult = Mockito.mock(PolicyNatRuleListResult.class); + PolicyNatRuleListResult remainingNatRulesResult = Mockito.mock(PolicyNatRuleListResult.class); + IPSecVpnServiceListResult vpnServicesResult = Mockito.mock(IPSecVpnServiceListResult.class); + com.vmware.nsx_policy.model.StaticRoutes vpnStaticRoute = + Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + com.vmware.nsx_policy.model.StaticRoutes operatorStaticRoute = + Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + PolicyNatRule vpnNoSnatRule = Mockito.mock(PolicyNatRule.class); + PolicyNatRule operatorNatRule = Mockito.mock(PolicyNatRule.class); + + when(nsxService.apply(Tier1s.class)).thenReturn(tier1s); + when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + when(nsxService.apply(NatRules.class)).thenReturn(natRules); + when(nsxService.apply(IpsecVpnServices.class)).thenReturn(vpnServices); + when(nsxService.apply(LocaleServices.class)).thenReturn(localeServices); + when(tier1s.get(TIER_1_GATEWAY_NAME)).thenReturn(Mockito.mock(Tier1.class)); + when(staticRoutes.list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(staticRoutesResult); + when(vpnStaticRoute.getId()).thenReturn("cs-conn-connection-uuid-route0"); + when(operatorStaticRoute.getId()).thenReturn("operator-route"); + when(staticRoutesResult.getResults()).thenReturn(List.of(vpnStaticRoute, operatorStaticRoute)); + when(natRules.list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(natRulesResult, remainingNatRulesResult); + when(vpnNoSnatRule.getId()).thenReturn("cs-conn-connection-uuid-nosnat0"); + when(operatorNatRule.getId()).thenReturn("operator-nat-rule"); + when(natRulesResult.getResults()).thenReturn(List.of(vpnNoSnatRule, operatorNatRule)); + when(remainingNatRulesResult.getResults()).thenReturn(List.of(operatorNatRule)); + when(vpnServices.list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), eq(false), nullable(String.class))) + .thenReturn(vpnServicesResult); + when(vpnServicesResult.getResults()).thenReturn(List.of()); + + client.deleteTier1Gateway(TIER_1_GATEWAY_NAME); + + InOrder inOrder = Mockito.inOrder(staticRoutes, natRules, vpnServices, localeServices, tier1s); + inOrder.verify(staticRoutes).list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + inOrder.verify(staticRoutes).delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route0"); + inOrder.verify(natRules).list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + inOrder.verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat0"); + inOrder.verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "t1-vpn-le-nosnat"); + inOrder.verify(vpnServices).list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), eq(false), nullable(String.class)); + inOrder.verify(natRules).list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + inOrder.verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "operator-nat-rule"); + inOrder.verify(localeServices).delete(TIER_1_GATEWAY_NAME, "default"); + inOrder.verify(tier1s).delete(TIER_1_GATEWAY_NAME); + verify(staticRoutes, never()).delete(TIER_1_GATEWAY_NAME, "operator-route"); + } + + @Test + public void testCreateRouteBasedVpnSessionRemovesSessionWhenPatchFails() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + Structure errorData = Mockito.mock(Structure.class); + ApiError apiError = new ApiError(); + apiError.setErrorData(errorData); + + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(errorData._convertTo(ApiError.class)).thenReturn(apiError); + doThrow(new Error(List.of(), errorData)).when(sessions).patch(anyString(), anyString(), anyString(), any(RouteBasedIPSecVpnSession.class)); + + assertThrows(CloudRuntimeException.class, () -> client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "connection-uuid", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.21", 30)); + + verify(sessions).delete(eq(TIER_1_GATEWAY_NAME), eq("t1-vpn"), eq("cs-conn-connection-uuid")); + verify(ikeProfiles).delete("cs-conn-connection-uuid-ike"); + verify(tunnelProfiles).delete("cs-conn-connection-uuid-esp"); + verify(dpdProfiles).delete("cs-conn-connection-uuid-dpd"); + } + + @Test + public void testCreateRouteBasedVpnSessionLeavesExistingSessionDisabledWhenPatchFails() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + RouteBasedIPSecVpnSession existingSession = createCompleteVpnSession("secret-psk"); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + Structure errorData = Mockito.mock(Structure.class); + ApiError apiError = new ApiError(); + apiError.setErrorData(errorData); + + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.get(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(existingSession); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(existingSession); + mockEmptyVpnConnectionRouteLists(staticRoutes, natRules); + Mockito.when(errorData._convertTo(ApiError.class)).thenReturn(apiError); + doThrow(new Error(List.of(), errorData)).when(sessions).patch(anyString(), anyString(), anyString(), any(RouteBasedIPSecVpnSession.class)); + + assertThrows(CloudRuntimeException.class, () -> client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "connection-uuid", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.21", 30)); + + verify(sessions, never()).delete(anyString(), anyString(), anyString()); + verify(ikeProfiles, never()).delete(anyString()); + verify(tunnelProfiles, never()).delete(anyString()); + verify(dpdProfiles, never()).delete(anyString()); + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass(Structure.class); + InOrder inOrder = Mockito.inOrder(sessions, ikeProfiles); + inOrder.verify(sessions).update(eq(TIER_1_GATEWAY_NAME), eq("t1-vpn"), + eq("cs-conn-connection-uuid"), updateCaptor.capture()); + inOrder.verify(ikeProfiles).patch(eq("cs-conn-connection-uuid-ike"), any(IPSecVpnIkeProfile.class)); + RouteBasedIPSecVpnSession update = updateCaptor.getValue()._convertTo(RouteBasedIPSecVpnSession.class); + assertFalse(update.getEnabled()); + } + + @Test + public void testCreateRouteBasedVpnSessionCleansProfilesWhenProfilePatchFails() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + Structure errorData = Mockito.mock(Structure.class); + ApiError apiError = new ApiError(); + apiError.setErrorData(errorData); + + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(errorData._convertTo(ApiError.class)).thenReturn(apiError); + doThrow(new Error(List.of(), errorData)).when(tunnelProfiles) + .patch(anyString(), any(IPSecVpnTunnelProfile.class)); + + assertThrows(CloudRuntimeException.class, () -> client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "connection-uuid", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.21", 30)); + + verify(sessions).delete(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid"); + verify(ikeProfiles).delete("cs-conn-connection-uuid-ike"); + verify(tunnelProfiles).delete("cs-conn-connection-uuid-esp"); + verify(dpdProfiles).delete("cs-conn-connection-uuid-dpd"); + } + + @Test + public void testCreateRouteBasedVpnSessionPreservesPreExistingProfilesAfterFailure() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + Structure errorData = Mockito.mock(Structure.class); + ApiError apiError = new ApiError(); + apiError.setErrorData(errorData); + + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(ikeProfiles.get("cs-conn-connection-uuid-ike")).thenReturn(Mockito.mock(IPSecVpnIkeProfile.class)); + Mockito.when(tunnelProfiles.get("cs-conn-connection-uuid-esp")).thenReturn(Mockito.mock(IPSecVpnTunnelProfile.class)); + Mockito.when(dpdProfiles.get("cs-conn-connection-uuid-dpd")).thenReturn(Mockito.mock(IPSecVpnDpdProfile.class)); + Mockito.when(errorData._convertTo(ApiError.class)).thenReturn(apiError); + doThrow(new Error(List.of(), errorData)).when(sessions) + .patch(anyString(), anyString(), anyString(), any(RouteBasedIPSecVpnSession.class)); + + assertThrows(CloudRuntimeException.class, () -> client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "connection-uuid", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.21", 30)); + + verify(ikeProfiles, never()).delete(anyString()); + verify(tunnelProfiles, never()).delete(anyString()); + verify(dpdProfiles, never()).delete(anyString()); + } + + @Test + public void testCreateRouteBasedVpnSessionReportsWhetherSessionWasPreexisting() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + + assertEquals(NsxApiClient.VpnSessionProvisioningResult.CREATED, client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "new-connection", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.21", 30)); + + Mockito.when(sessions.get(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-existing-connection")) + .thenReturn(Mockito.mock(Structure.class)); + RouteBasedIPSecVpnSession existingSession = createCompleteVpnSession("secret-psk"); + existingSession.setId("cs-conn-existing-connection"); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-existing-connection")) + .thenReturn(existingSession); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + mockEmptyVpnConnectionRouteLists(staticRoutes, natRules); + assertEquals(NsxApiClient.VpnSessionProvisioningResult.PREEXISTING, client.createRouteBasedVpnSession( + TIER_1_GATEWAY_NAME, "existing-connection", "203.0.113.10", "psk", + "aes256-sha256;modp2048", "aes256-sha256;modp2048", 86400L, 3600L, + true, "ikev2", false, "169.254.64.25", 30)); + + ArgumentCaptor sessionCaptor = ArgumentCaptor.forClass(RouteBasedIPSecVpnSession.class); + verify(sessions, Mockito.times(2)).patch(eq(TIER_1_GATEWAY_NAME), eq("t1-vpn"), anyString(), sessionCaptor.capture()); + assertTrue(sessionCaptor.getAllValues().stream().noneMatch(RouteBasedIPSecVpnSession::getEnabled)); + ArgumentCaptor dpdProfileCaptor = ArgumentCaptor.forClass(IPSecVpnDpdProfile.class); + verify(dpdProfiles, Mockito.times(2)).patch(anyString(), dpdProfileCaptor.capture()); + assertTrue(dpdProfileCaptor.getAllValues().stream().allMatch(profile -> + IPSecVpnDpdProfile.DPD_PROBE_MODE_ON_DEMAND.equals(profile.getDpdProbeMode()) + && Long.valueOf(10L).equals(profile.getDpdProbeInterval()) + && Long.valueOf(10L).equals(profile.getRetryCount()))); + } + + @Test + public void testUpdateVpnConnectionStateUsesSensitiveFullReplace() { + Sessions sessions = Mockito.mock(Sessions.class); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + RouteBasedIPSecVpnSession session = createCompleteVpnSession("secret-psk"); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(session); + mockEmptyVpnConnectionRouteLists(staticRoutes, natRules); + + client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false); + + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass(Structure.class); + InOrder inOrder = Mockito.inOrder(sessions, staticRoutes, natRules); + inOrder.verify(sessions).showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid"); + inOrder.verify(sessions).update(eq(TIER_1_GATEWAY_NAME), eq("t1-vpn"), + eq("cs-conn-connection-uuid"), updateCaptor.capture()); + inOrder.verify(staticRoutes).list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + inOrder.verify(natRules).list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + RouteBasedIPSecVpnSession update = updateCaptor.getValue()._convertTo(RouteBasedIPSecVpnSession.class); + assertFalse(update.getEnabled()); + assertEquals("secret-psk", update.getPsk()); + assertEquals("203.0.113.10", update.getPeerAddress()); + assertEquals("/infra/tier-1s/t1/ipsec-vpn-services/t1-vpn/local-endpoints/t1-vpn-le", update.getLocalEndpointPath()); + assertEquals("/infra/ipsec-vpn-ike-profiles/ike", update.getIkeProfilePath()); + assertEquals("/infra/ipsec-vpn-tunnel-profiles/esp", update.getTunnelProfilePath()); + assertEquals("/infra/ipsec-vpn-dpd-profiles/dpd", update.getDpdProfilePath()); + assertEquals(Long.valueOf(7L), update.getRevision()); + assertEquals(1, update.getTunnelInterfaces().size()); + verify(sessions, never()).patch(anyString(), anyString(), anyString(), any(Structure.class)); + } + + @Test + public void testUpdateVpnConnectionStateDoesNotCleanupWhenPutFails() { + Sessions sessions = Mockito.mock(Sessions.class); + Structure errorData = Mockito.mock(Structure.class); + ApiError apiError = new ApiError(); + apiError.setErrorMessage("update failed"); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(createCompleteVpnSession("secret-psk")); + Mockito.when(errorData._convertTo(ApiError.class)).thenReturn(apiError); + doThrow(new Error(List.of(), errorData)).when(sessions) + .update(anyString(), anyString(), anyString(), any(Structure.class)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false)); + + assertFalse(exception.getMessage().contains("secret-psk")); + verify(nsxService, never()).apply(StaticRoutes.class); + verify(nsxService, never()).apply(NatRules.class); + } + + @Test + public void testUpdateVpnConnectionStateRejectsMissingSensitivePsk() { + Sessions sessions = Mockito.mock(Sessions.class); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(createCompleteVpnSession(null)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false)); + + assertTrue(exception.getMessage().contains("did not return sensitive authentication data")); + verify(sessions, never()).update(anyString(), anyString(), anyString(), any(Structure.class)); + verify(nsxService, never()).apply(StaticRoutes.class); + verify(nsxService, never()).apply(NatRules.class); + } + + @Test + public void testUpdateVpnConnectionStateRejectsNonRouteBasedSession() { + Sessions sessions = Mockito.mock(Sessions.class); + Structure session = Mockito.mock(Structure.class); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(session); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false)); + + assertTrue(exception.getMessage().contains("is not route-based")); + verify(sessions, never()).update(anyString(), anyString(), anyString(), any(Structure.class)); + verify(nsxService, never()).apply(StaticRoutes.class); + verify(nsxService, never()).apply(NatRules.class); + } + + @Test + public void testUpdateVpnConnectionStateRejectsMissingRevision() { + Sessions sessions = Mockito.mock(Sessions.class); + RouteBasedIPSecVpnSession session = createCompleteVpnSession("secret-psk"); + session.setRevision(null); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenReturn(session); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false)); + + assertTrue(exception.getMessage().contains("returned no revision")); + verify(sessions, never()).update(anyString(), anyString(), anyString(), any(Structure.class)); + verify(nsxService, never()).apply(StaticRoutes.class); + verify(nsxService, never()).apply(NatRules.class); + } + + @Test + public void testDisableMissingVpnSessionStillCleansStaleRoutesAndNat() { + Sessions sessions = Mockito.mock(Sessions.class); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenThrow(new NotFound(null, null)); + mockEmptyVpnConnectionRouteLists(staticRoutes, natRules); + + client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false); + + verify(sessions, never()).update(anyString(), anyString(), anyString(), any(Structure.class)); + verify(staticRoutes).list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + verify(natRules).list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class)); + } + + @Test + public void testEnableMissingVpnSessionFailsWithoutRouteCleanup() { + Sessions sessions = Mockito.mock(Sessions.class); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenThrow(new NotFound(null, null)); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", true)); + + assertTrue(exception.getMessage().contains("because it does not exist")); + verify(sessions, never()).update(anyString(), anyString(), anyString(), any(Structure.class)); + verify(nsxService, never()).apply(StaticRoutes.class); + verify(nsxService, never()).apply(NatRules.class); + } + + @Test + public void testVpnRouteCleanupContinuesWhenIndividualObjectsAreAlreadyAbsent() { + Sessions sessions = Mockito.mock(Sessions.class); + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + StaticRoutesListResult routeList = Mockito.mock(StaticRoutesListResult.class); + PolicyNatRuleListResult ruleList = Mockito.mock(PolicyNatRuleListResult.class); + com.vmware.nsx_policy.model.StaticRoutes firstRoute = Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + com.vmware.nsx_policy.model.StaticRoutes secondRoute = Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + PolicyNatRule firstRule = Mockito.mock(PolicyNatRule.class); + PolicyNatRule secondRule = Mockito.mock(PolicyNatRule.class); + Mockito.when(firstRoute.getId()).thenReturn("cs-conn-connection-uuid-route0"); + Mockito.when(secondRoute.getId()).thenReturn("cs-conn-connection-uuid-route1"); + Mockito.when(firstRule.getId()).thenReturn("cs-conn-connection-uuid-nosnat0"); + Mockito.when(secondRule.getId()).thenReturn("cs-conn-connection-uuid-nosnat1"); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(sessions.showsensitivedata(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid")) + .thenThrow(new NotFound(null, null)); + Mockito.when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + Mockito.when(staticRoutes.list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(routeList); + Mockito.when(routeList.getResults()).thenReturn(List.of(firstRoute, secondRoute)); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + Mockito.when(natRules.list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(ruleList); + Mockito.when(ruleList.getResults()).thenReturn(List.of(firstRule, secondRule)); + doThrow(new NotFound(null, null)).when(staticRoutes) + .delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route0"); + doThrow(new NotFound(null, null)).when(natRules) + .delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat0"); + + client.updateVpnConnectionState(TIER_1_GATEWAY_NAME, "connection-uuid", false); + + verify(staticRoutes).delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route0"); + verify(staticRoutes).delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route1"); + verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat0"); + verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat1"); + } + + @Test + public void testAddVpnConnectionRoutesPatchesDesiredBeforeDeletingStale() { + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + StaticRoutesListResult routeList = Mockito.mock(StaticRoutesListResult.class); + PolicyNatRuleListResult ruleList = Mockito.mock(PolicyNatRuleListResult.class); + com.vmware.nsx_policy.model.StaticRoutes desiredRoute = Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + com.vmware.nsx_policy.model.StaticRoutes staleRoute = Mockito.mock(com.vmware.nsx_policy.model.StaticRoutes.class); + PolicyNatRule desiredRule = Mockito.mock(PolicyNatRule.class); + PolicyNatRule staleRule = Mockito.mock(PolicyNatRule.class); + Mockito.when(desiredRoute.getId()).thenReturn("cs-conn-connection-uuid-route0"); + Mockito.when(staleRoute.getId()).thenReturn("cs-conn-connection-uuid-route1"); + Mockito.when(desiredRule.getId()).thenReturn("cs-conn-connection-uuid-nosnat0"); + Mockito.when(staleRule.getId()).thenReturn("cs-conn-connection-uuid-nosnat1"); + Mockito.when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + Mockito.when(staticRoutes.list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(routeList); + Mockito.when(routeList.getResults()).thenReturn(List.of(desiredRoute, staleRoute)); + Mockito.when(natRules.list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(ruleList); + Mockito.when(ruleList.getResults()).thenReturn(List.of(desiredRule, staleRule)); + + client.addVpnConnectionRoutes(TIER_1_GATEWAY_NAME, "connection-uuid", + List.of("192.168.100.0/24"), "169.254.64.22", "10.1.0.0/16"); + + InOrder inOrder = Mockito.inOrder(staticRoutes, natRules); + inOrder.verify(staticRoutes).patch(eq(TIER_1_GATEWAY_NAME), eq("cs-conn-connection-uuid-route0"), + any(com.vmware.nsx_policy.model.StaticRoutes.class)); + inOrder.verify(natRules).patch(eq(TIER_1_GATEWAY_NAME), anyString(), + eq("cs-conn-connection-uuid-nosnat0"), any(PolicyNatRule.class)); + inOrder.verify(staticRoutes).delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route1"); + inOrder.verify(natRules).delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat1"); + verify(staticRoutes, never()).delete(TIER_1_GATEWAY_NAME, "cs-conn-connection-uuid-route0"); + verify(natRules, never()).delete(TIER_1_GATEWAY_NAME, "USER", "cs-conn-connection-uuid-nosnat0"); + } + + @Test + public void testAddVpnConnectionRoutesPatchFailureDoesNotDeleteExistingResources() { + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + Mockito.when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + doThrow(new CloudRuntimeException("route patch failed")).when(staticRoutes) + .patch(anyString(), anyString(), any(com.vmware.nsx_policy.model.StaticRoutes.class)); + + assertThrows(CloudRuntimeException.class, () -> client.addVpnConnectionRoutes(TIER_1_GATEWAY_NAME, + "connection-uuid", List.of("192.168.100.0/24"), "169.254.64.22", "10.1.0.0/16")); + + verify(staticRoutes, never()).list(anyString(), any(), anyBoolean(), any(), any(), any(), any()); + verify(natRules, never()).list(anyString(), anyString(), any(), anyBoolean(), any(), any(), any(), any()); + verify(staticRoutes, never()).delete(anyString(), anyString()); + verify(natRules, never()).delete(anyString(), anyString(), anyString()); + } + + @Test + public void testAddVpnConnectionRoutesRetriesMarkedForDeletion() { + StaticRoutes staticRoutes = Mockito.mock(StaticRoutes.class); + NatRules natRules = Mockito.mock(NatRules.class); + Mockito.when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + mockEmptyVpnConnectionRouteLists(staticRoutes, natRules); + doThrow(new CloudRuntimeException("An object is marked for deletion")) + .doNothing() + .when(staticRoutes).patch(anyString(), anyString(), any(com.vmware.nsx_policy.model.StaticRoutes.class)); + + client.addVpnConnectionRoutes(TIER_1_GATEWAY_NAME, "connection-uuid", + List.of("192.168.100.0/24"), "169.254.64.22", "10.1.0.0/16"); + + verify(staticRoutes, Mockito.times(2)).patch(eq(TIER_1_GATEWAY_NAME), + eq("cs-conn-connection-uuid-route0"), any(com.vmware.nsx_policy.model.StaticRoutes.class)); + verify(natRules).patch(eq(TIER_1_GATEWAY_NAME), anyString(), + eq("cs-conn-connection-uuid-nosnat0"), any(PolicyNatRule.class)); + } + + @Test + public void testDeleteVpnConnectionContinuesCleanupAfterRouteAndNatFailures() { + IpsecVpnIkeProfiles ikeProfiles = Mockito.mock(IpsecVpnIkeProfiles.class); + IpsecVpnTunnelProfiles tunnelProfiles = Mockito.mock(IpsecVpnTunnelProfiles.class); + IpsecVpnDpdProfiles dpdProfiles = Mockito.mock(IpsecVpnDpdProfiles.class); + Sessions sessions = Mockito.mock(Sessions.class); + Mockito.when(nsxService.apply(com.vmware.nsx_policy.infra.tier_1s.StaticRoutes.class)) + .thenThrow(new CloudRuntimeException("route cleanup failed")); + Mockito.when(nsxService.apply(NatRules.class)).thenThrow(new CloudRuntimeException("NAT cleanup failed")); + Mockito.when(nsxService.apply(Sessions.class)).thenReturn(sessions); + Mockito.when(nsxService.apply(IpsecVpnIkeProfiles.class)).thenReturn(ikeProfiles); + Mockito.when(nsxService.apply(IpsecVpnTunnelProfiles.class)).thenReturn(tunnelProfiles); + Mockito.when(nsxService.apply(IpsecVpnDpdProfiles.class)).thenReturn(dpdProfiles); + + assertThrows(CloudRuntimeException.class, + () -> client.deleteVpnConnection(TIER_1_GATEWAY_NAME, "connection-uuid")); + + verify(sessions).delete(TIER_1_GATEWAY_NAME, "t1-vpn", "cs-conn-connection-uuid"); + verify(ikeProfiles).delete("cs-conn-connection-uuid-ike"); + verify(tunnelProfiles).delete("cs-conn-connection-uuid-esp"); + verify(dpdProfiles).delete("cs-conn-connection-uuid-dpd"); + } + private LbMonitorProfiles mockLbMonitorProfiles() { LbMonitorProfiles lbMonitorProfiles = Mockito.mock(LbMonitorProfiles.class); Structure monitorStructure = Mockito.mock(Structure.class, Mockito.RETURNS_DEEP_STUBS); @@ -401,6 +1309,49 @@ private LbMonitorProfiles mockLbMonitorProfiles() { return lbMonitorProfiles; } + private RouteBasedIPSecVpnSession createCompleteVpnSession(String psk) { + IPSecVpnTunnelInterface tunnelInterface = new IPSecVpnTunnelInterface.Builder() + .setId("default-tunnel-interface") + .setDisplayName("default-tunnel-interface") + .setIpSubnets(List.of(new TunnelInterfaceIPSubnet.Builder() + .setIpAddresses(List.of("169.254.64.21")) + .setPrefixLength(30L) + .build())) + .build(); + RouteBasedIPSecVpnSession session = new RouteBasedIPSecVpnSession.Builder() + .setId("cs-conn-connection-uuid") + .setDisplayName("cs-conn-connection-uuid") + .setEnabled(true) + .setAuthenticationMode(IPSecVpnSession.AUTHENTICATION_MODE_PSK) + .setPsk(psk) + .setPeerAddress("203.0.113.10") + .setPeerId("203.0.113.10") + .setConnectionInitiationMode(IPSecVpnSession.CONNECTION_INITIATION_MODE_INITIATOR) + .setIkeProfilePath("/infra/ipsec-vpn-ike-profiles/ike") + .setTunnelProfilePath("/infra/ipsec-vpn-tunnel-profiles/esp") + .setDpdProfilePath("/infra/ipsec-vpn-dpd-profiles/dpd") + .setLocalEndpointPath("/infra/tier-1s/t1/ipsec-vpn-services/t1-vpn/local-endpoints/t1-vpn-le") + .setTunnelInterfaces(List.of(tunnelInterface)) + .build(); + session.setRevision(7L); + return session; + } + + private void mockEmptyVpnConnectionRouteLists(StaticRoutes staticRoutes, NatRules natRules) { + StaticRoutesListResult routeList = Mockito.mock(StaticRoutesListResult.class); + PolicyNatRuleListResult ruleList = Mockito.mock(PolicyNatRuleListResult.class); + Mockito.when(nsxService.apply(StaticRoutes.class)).thenReturn(staticRoutes); + Mockito.when(nsxService.apply(NatRules.class)).thenReturn(natRules); + Mockito.when(staticRoutes.list(eq(TIER_1_GATEWAY_NAME), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(routeList); + Mockito.when(routeList.getResults()).thenReturn(List.of()); + Mockito.when(natRules.list(eq(TIER_1_GATEWAY_NAME), anyString(), nullable(String.class), eq(false), + nullable(String.class), nullable(Long.class), nullable(Boolean.class), nullable(String.class))) + .thenReturn(ruleList); + Mockito.when(ruleList.getResults()).thenReturn(List.of()); + } + private void mockLbAppProfiles() { LbAppProfiles lbAppProfiles = Mockito.mock(LbAppProfiles.class); LBAppProfileListResult appProfileListResult = Mockito.mock(LBAppProfileListResult.class); diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java index a807155a2dc1..8f5cc6128a73 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java @@ -23,36 +23,54 @@ import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.addr.PublicIp; import com.cloud.network.Network; import com.cloud.network.NetworkModel; import com.cloud.network.Networks; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.Site2SiteVpnGateway; +import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.network.dao.Site2SiteCustomerGatewayDao; +import com.cloud.network.dao.Site2SiteCustomerGatewayVO; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.Site2SiteVpnGatewayVO; import com.cloud.network.element.PortForwardingServiceProvider; import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.nsx.NsxVpnGatewayResult; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRule; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.StaticNatImpl; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.NetworkACLItemVO; import com.cloud.network.vpc.Vpc; +import com.cloud.network.vpc.VpcOfferingServiceMapVO; +import com.cloud.network.vpc.VpcService; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; import com.cloud.resource.ResourceManager; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.User; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.vm.NicVO; import com.cloud.vm.ReservationContext; @@ -61,8 +79,11 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.resource.NsxNetworkRule; +import org.apache.cloudstack.resourcedetail.UserIpAddressDetailVO; import org.apache.cloudstack.resourcedetail.dao.FirewallRuleDetailsDao; +import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -75,15 +96,22 @@ import java.lang.reflect.InvocationTargetException; import java.util.List; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -126,7 +154,23 @@ public class NsxElementTest { @Mock LoadBalancerVMMapDao lbVmMapDao; @Mock + LoadBalancerDao loadBalancerDao; + @Mock FirewallRuleDetailsDao firewallRuleDetailsDao; + @Mock + IpAddressManager ipAddressManager; + @Mock + VpcService vpcService; + @Mock + Site2SiteVpnGatewayDao vpnGatewayDao; + @Mock + Site2SiteCustomerGatewayDao customerGatewayDao; + @Mock + UserIpAddressDetailsDao userIpAddressDetailsDao; + @Mock + FirewallRulesDao firewallRulesDao; + @Mock + PortForwardingRulesDao portForwardingRulesDao; NsxElement nsxElement; ReservationContext reservationContext; @@ -151,7 +195,19 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { nsxElement.vmInstanceDao = vmInstanceDao; nsxElement.vpcDao = vpcDao; nsxElement.lbVmMapDao = lbVmMapDao; + nsxElement.loadBalancerDao = loadBalancerDao; nsxElement.firewallRuleDetailsDao = firewallRuleDetailsDao; + nsxElement.ipAddressManager = ipAddressManager; + nsxElement.vpcService = vpcService; + nsxElement.vpnGatewayDao = vpnGatewayDao; + nsxElement.customerGatewayDao = customerGatewayDao; + nsxElement.userIpAddressDetailsDao = userIpAddressDetailsDao; + nsxElement.firewallRulesDao = firewallRulesDao; + nsxElement.portForwardingRulesDao = portForwardingRulesDao; + Mockito.lenient().when(ipAddressManager.disassociatePublicIpAddress(any(), anyLong(), any())).thenReturn(true); + Mockito.lenient().when(loadBalancerDao.listByIpAddress(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(firewallRulesDao.listByIpAndNotRevoked(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(portForwardingRulesDao.listByIpAndNotRevoked(anyLong())).thenReturn(List.of()); Field field = ApiDBUtils.class.getDeclaredField("s_ipAddressDao"); field.setAccessible(true); @@ -470,4 +526,507 @@ public void testRevokeFirewallRules() throws ResourceUnavailableException { when(nsxService.addFirewallRules(any(Network.class), any(List.class))).thenReturn(true); assertTrue(nsxElement.applyFWRules(networkVO, List.of(rule))); } + + private VpcVO mockVpcWithNsxVpnSupport() { + VpcVO vpcVO = Mockito.mock(VpcVO.class); + Mockito.lenient().when(vpcVO.getId()).thenReturn(9L); + when(vpcVO.getVpcOfferingId()).thenReturn(11L); + when(vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.Vpn.getName(), + Network.Provider.Nsx.getName(), 11L)).thenReturn(Mockito.mock(VpcOfferingServiceMapVO.class)); + return vpcVO; + } + + private IPAddressVO mockIpAddressVO(long id, String address) { + IPAddressVO ipAddressVO = Mockito.mock(IPAddressVO.class); + when(ipAddressDao.findById(id)).thenReturn(ipAddressVO); + Mockito.lenient().when(ipAddressVO.getAddress()).thenReturn(new Ip(address)); + Mockito.lenient().when(ipAddressVO.readyToUse()).thenReturn(true); + Mockito.lenient().when(ipAddressVO.getRemoved()).thenReturn(null); + return ipAddressVO; + } + + @Test + public void testAcquireVpnGatewayIpWithRequestedIp() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = mockIpAddressVO(20L, "10.1.13.20"); + when(ipAddressVO.getId()).thenReturn(20L); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(loadBalancerDao.listByIpAddress(20L)).thenReturn(List.of()); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.20")).thenReturn(new NsxVpnGatewayResult(true, true)); + + IpAddress result = nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp); + assertEquals(ipAddressVO, result); + verify(userIpAddressDetailsDao).addDetail(20L, "nsxVpnGatewayIp", "false", false); + } + + @Test + public void testAcquireVpnGatewayIpDoesNotCallNsxWhenRequestedIpOwnershipCannotBeRecorded() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = mockIpAddressVO(20L, "10.1.13.20"); + when(ipAddressVO.getId()).thenReturn(20L); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(loadBalancerDao.listByIpAddress(20L)).thenReturn(List.of()); + Mockito.doThrow(new CloudRuntimeException("marker write failed")).when(userIpAddressDetailsDao) + .addDetail(20L, "nsxVpnGatewayIp", "false", false); + + Assert.assertThrows(CloudRuntimeException.class, + () -> nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp)); + + verify(nsxService, never()).createVpnGateway(any(Vpc.class), anyString()); + } + + @Test + public void testAcquireVpnGatewayIpRemovesRequestedIpOwnershipWhenNsxRejectsGateway() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = mockIpAddressVO(20L, "10.1.13.20"); + when(ipAddressVO.getId()).thenReturn(20L); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(loadBalancerDao.listByIpAddress(20L)).thenReturn(List.of()); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.20")).thenReturn(new NsxVpnGatewayResult(false, false)); + + Assert.assertThrows(CloudRuntimeException.class, + () -> nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp)); + + verify(userIpAddressDetailsDao).removeDetail(20L, "nsxVpnGatewayIp"); + } + + @Test + public void testAcquireVpnGatewayIpRetainsRequestedIpOwnershipWhenNsxResultIsAmbiguous() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = mockIpAddressVO(20L, "10.1.13.20"); + when(ipAddressVO.getId()).thenReturn(20L); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(loadBalancerDao.listByIpAddress(20L)).thenReturn(List.of()); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.20")).thenReturn(new NsxVpnGatewayResult(false, true)); + + Assert.assertThrows(CloudRuntimeException.class, + () -> nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp)); + + verify(userIpAddressDetailsDao, never()).removeDetail(20L, "nsxVpnGatewayIp"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testAcquireVpnGatewayIpRejectsSourceNatIp() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = Mockito.mock(IPAddressVO.class); + when(ipAddressDao.findById(20L)).thenReturn(ipAddressVO); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(ipAddressVO.readyToUse()).thenReturn(true); + when(ipAddressVO.getRemoved()).thenReturn(null); + when(ipAddressVO.isSourceNat()).thenReturn(true); + when(ipAddressVO.getAddress()).thenReturn(new Ip("10.1.13.20")); + + nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp); + } + + @Test(expected = InvalidParameterValueException.class) + public void testAcquireVpnGatewayIpRejectsIpWithPortForwardingRule() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(20L); + IPAddressVO ipAddressVO = mockIpAddressVO(20L, "10.1.13.20"); + when(ipAddressVO.getId()).thenReturn(20L); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(portForwardingRulesDao.listByIpAndNotRevoked(20L)) + .thenReturn(List.of(Mockito.mock(PortForwardingRuleVO.class))); + + nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp); + } + + @Test(expected = InvalidParameterValueException.class) + public void testAcquireVpnGatewayIpRejectsAnUnallocatedIp() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + IpAddress requestedIp = Mockito.mock(IpAddress.class); + when(requestedIp.getId()).thenReturn(21L); + IPAddressVO ipAddressVO = Mockito.mock(IPAddressVO.class); + when(ipAddressDao.findById(21L)).thenReturn(ipAddressVO); + when(ipAddressVO.getVpcId()).thenReturn(9L); + when(ipAddressVO.readyToUse()).thenReturn(false); + + nsxElement.acquireVpnGatewayIp(vpcVO, requestedIp); + } + + @Test + public void testAcquireVpnGatewayIpReturnsNullWhenVpnIsNotProvidedByNsx() { + VpcVO vpcVO = Mockito.mock(VpcVO.class); + when(vpcVO.getVpcOfferingId()).thenReturn(11L); + + assertNull(nsxElement.acquireVpnGatewayIp(vpcVO, null)); + } + + @Test + public void testAcquireVpnGatewayIpAutoAcquiresWhenNoIpIsRequested() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcVO.getAccountId()).thenReturn(2L); + when(vpcVO.getZoneId()).thenReturn(1L); + when(nsxService.getPublicVlanId(anyLong(), anyLong(), anyLong(), anyLong(), any())).thenReturn(null); + IpAddress allocatedIp = Mockito.mock(IpAddress.class); + when(allocatedIp.getId()).thenReturn(30L); + when(ipAddressManager.allocateIp(any(), anyBoolean(), any(), any(), any(), any(), any())).thenReturn(allocatedIp); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.30")).thenReturn(new NsxVpnGatewayResult(true, true)); + + IpAddress result = nsxElement.acquireVpnGatewayIp(vpcVO, null); + assertEquals(ipAddressVO, result); + verify(vpcService).associateIPToVpc(30L, 9L); + verify(userIpAddressDetailsDao).addDetail(30L, "nsxVpnGatewayIp", "true", false); + } finally { + CallContext.unregister(); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testAcquireVpnGatewayIpReleasesAutoAcquiredIpWhenNsxRejectsGateway() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcVO.getAccountId()).thenReturn(2L); + when(vpcVO.getZoneId()).thenReturn(1L); + when(nsxService.getPublicVlanId(anyLong(), anyLong(), anyLong(), anyLong(), any())).thenReturn(null); + IpAddress allocatedIp = Mockito.mock(IpAddress.class); + when(allocatedIp.getId()).thenReturn(31L); + when(ipAddressManager.allocateIp(any(), anyBoolean(), any(), any(), any(), any(), any())).thenReturn(allocatedIp); + IPAddressVO ipAddressVO = mockIpAddressVO(31L, "10.1.13.31"); + when(ipAddressVO.getId()).thenReturn(31L); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.31")).thenReturn(new NsxVpnGatewayResult(false, false)); + when(ipAddressManager.disassociatePublicIpAddress(any(IPAddressVO.class), anyLong(), any())).thenReturn(true); + + nsxElement.acquireVpnGatewayIp(vpcVO, null); + } finally { + verify(userIpAddressDetailsDao).removeDetail(31L, "nsxVpnGatewayIp"); + verify(ipAddressManager).disassociatePublicIpAddress(any(IPAddressVO.class), anyLong(), any()); + CallContext.unregister(); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testAcquireVpnGatewayIpRetainsAutoAcquiredIpWhenEndpointMayBeInUse() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcVO.getAccountId()).thenReturn(2L); + when(vpcVO.getZoneId()).thenReturn(1L); + when(nsxService.getPublicVlanId(anyLong(), anyLong(), anyLong(), anyLong(), any())).thenReturn(null); + IpAddress allocatedIp = Mockito.mock(IpAddress.class); + when(allocatedIp.getId()).thenReturn(32L); + when(ipAddressManager.allocateIp(any(), anyBoolean(), any(), any(), any(), any(), any())).thenReturn(allocatedIp); + mockIpAddressVO(32L, "10.1.13.32"); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.32")).thenReturn(new NsxVpnGatewayResult(false, true)); + + nsxElement.acquireVpnGatewayIp(vpcVO, null); + } finally { + verify(userIpAddressDetailsDao, never()).removeDetail(32L, "nsxVpnGatewayIp"); + verify(ipAddressManager, never()).disassociatePublicIpAddress(any(IPAddressVO.class), anyLong(), any()); + CallContext.unregister(); + } + } + + @Test + public void testAcquireVpnGatewayIpUsesRecordedVrfPublicRangeWithoutFallback() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcVO.getAccountId()).thenReturn(2L); + when(vpcVO.getZoneId()).thenReturn(1L); + when(nsxService.getPublicVlanId(anyLong(), anyLong(), anyLong(), anyLong(), any())).thenReturn(73L); + PublicIp allocatedIp = Mockito.mock(PublicIp.class); + when(allocatedIp.getId()).thenReturn(33L); + when(ipAddressManager.assignDedicateIpAddressFromNsxVrfPublicRange(eq(account), isNull(), + eq(9L), eq(1L), eq(false), eq(73L))).thenReturn(allocatedIp); + IPAddressVO ipAddressVO = mockIpAddressVO(33L, "10.1.13.33"); + when(ipAddressVO.getVlanId()).thenReturn(73L); + when(nsxService.createVpnGateway(vpcVO, "10.1.13.33")) + .thenReturn(new NsxVpnGatewayResult(true, false)); + + IpAddress result = nsxElement.acquireVpnGatewayIp(vpcVO, null); + + assertEquals(ipAddressVO, result); + verify(ipAddressManager).assignDedicateIpAddressFromNsxVrfPublicRange(eq(account), isNull(), + eq(9L), eq(1L), eq(false), eq(73L)); + verify(ipAddressManager, never()).allocateIp(any(), anyBoolean(), any(), any(), any(), any(), any()); + verify(vpcService, never()).associateIPToVpc(anyLong(), anyLong()); + verify(userIpAddressDetailsDao).addDetail(33L, "nsxVpnGatewayIp", "true", false); + } finally { + CallContext.unregister(); + } + } + + @Test + public void testReleaseVpnGatewayIpReleasesAutoAcquiredIp() { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcDao.findById(9L)).thenReturn(vpcVO); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + when(nsxService.deleteVpnGateway(vpcVO)).thenReturn(true); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(ipAddressVO.getId()).thenReturn(30L); + UserIpAddressDetailVO detail = Mockito.mock(UserIpAddressDetailVO.class); + when(detail.getValue()).thenReturn("true"); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(detail); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + verify(userIpAddressDetailsDao).removeDetail(30L, "nsxVpnGatewayIp"); + verify(ipAddressManager).disassociatePublicIpAddress(eq(ipAddressVO), anyLong(), any()); + } finally { + CallContext.unregister(); + } + } + + @Test + public void testReleaseVpnGatewayIpKeepsOperatorSpecifiedIp() { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcDao.findById(9L)).thenReturn(vpcVO); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + when(nsxService.deleteVpnGateway(vpcVO)).thenReturn(true); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(ipAddressVO.getId()).thenReturn(30L); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(null); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + verify(ipAddressManager, Mockito.never()).disassociatePublicIpAddress(any(), anyLong(), any()); + } + + @Test + public void testNsxVpnGatewayOwnershipDoesNotDependOnMarkerValue() { + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getAddrId()).thenReturn(30L); + UserIpAddressDetailVO operatorSpecified = new UserIpAddressDetailVO(30L, "nsxVpnGatewayIp", "false", false); + UserIpAddressDetailVO autoAcquired = new UserIpAddressDetailVO(30L, "nsxVpnGatewayIp", "true", false); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")) + .thenReturn(operatorSpecified, autoAcquired); + + assertTrue(nsxElement.ownsVpnGateway(vpnGateway)); + assertTrue(nsxElement.ownsVpnGateway(vpnGateway)); + } + + @Test + public void testNsxVpnGatewayOwnershipRequiresMarker() { + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getAddrId()).thenReturn(30L); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(null); + + assertFalse(nsxElement.ownsVpnGateway(vpnGateway)); + } + + @Test + public void testReleaseVpnGatewayIpUsesPersistedOwnershipWhenOfferingMappingIsGone() { + VpcVO vpcVO = Mockito.mock(VpcVO.class); + when(vpcDao.findById(9L)).thenReturn(vpcVO); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + when(nsxService.deleteVpnGateway(vpcVO)).thenReturn(true); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(ipAddressVO.getId()).thenReturn(30L); + UserIpAddressDetailVO detail = Mockito.mock(UserIpAddressDetailVO.class); + when(detail.getValue()).thenReturn("false"); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(detail); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + + verify(nsxService).deleteVpnGateway(vpcVO); + verify(userIpAddressDetailsDao).removeDetail(30L, "nsxVpnGatewayIp"); + verify(ipAddressManager, Mockito.never()).disassociatePublicIpAddress(any(), anyLong(), any()); + } + + @Test + public void testReleaseVpnGatewayIpRemovesOperatorMarkerWhenVpcRowIsGone() { + when(vpcDao.findById(9L)).thenReturn(null); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(ipAddressVO.getId()).thenReturn(30L); + UserIpAddressDetailVO detail = Mockito.mock(UserIpAddressDetailVO.class); + when(detail.getValue()).thenReturn("false"); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(detail); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + + verify(nsxService, Mockito.never()).deleteVpnGateway(any(Vpc.class)); + verify(userIpAddressDetailsDao).removeDetail(30L, "nsxVpnGatewayIp"); + verify(ipAddressManager, Mockito.never()).disassociatePublicIpAddress(any(), anyLong(), any()); + } + + @Test(expected = CloudRuntimeException.class) + public void testReleaseVpnGatewayIpDoesNotReleaseIpWhenNsxRejectsDeletion() { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcDao.findById(9L)).thenReturn(vpcVO); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(nsxService.deleteVpnGateway(vpcVO)).thenReturn(false); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + } finally { + verify(ipAddressManager, Mockito.never()).disassociatePublicIpAddress(any(), anyLong(), any()); + CallContext.unregister(); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testReleaseVpnGatewayIpKeepsTheMarkerWhenIpDisassociationFails() { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + when(vpcDao.findById(9L)).thenReturn(vpcVO); + Site2SiteVpnGateway vpnGateway = Mockito.mock(Site2SiteVpnGateway.class); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + when(nsxService.deleteVpnGateway(vpcVO)).thenReturn(true); + IPAddressVO ipAddressVO = mockIpAddressVO(30L, "10.1.13.30"); + when(ipAddressVO.getId()).thenReturn(30L); + UserIpAddressDetailVO detail = Mockito.mock(UserIpAddressDetailVO.class); + when(detail.getValue()).thenReturn("true"); + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")).thenReturn(detail); + when(ipAddressManager.disassociatePublicIpAddress(any(), anyLong(), any())).thenReturn(false); + + nsxElement.releaseVpnGatewayIp(vpnGateway); + } finally { + verify(userIpAddressDetailsDao, Mockito.never()).removeDetail(30L, "nsxVpnGatewayIp"); + CallContext.unregister(); + } + } + + private Site2SiteCustomerGatewayVO mockCustomerGateway(String ikePolicy, String espPolicy) { + Site2SiteCustomerGatewayVO customerGateway = Mockito.mock(Site2SiteCustomerGatewayVO.class); + when(customerGatewayDao.findById(3L)).thenReturn(customerGateway); + when(customerGateway.getIkePolicy()).thenReturn(ikePolicy); + when(customerGateway.getEspPolicy()).thenReturn(espPolicy); + when(customerGateway.getIkeVersion()).thenReturn("ikev2"); + when(customerGateway.getIkeLifetime()).thenReturn(86400L); + when(customerGateway.getEspLifetime()).thenReturn(3600L); + when(customerGateway.getIpsecPsk()).thenReturn("presharedkey"); + return customerGateway; + } + + private Site2SiteVpnConnection mockVpnConnection(VpcVO vpcVO) { + return mockVpnConnection(vpcVO, true); + } + + private Site2SiteVpnConnection mockVpnConnection(VpcVO vpcVO, boolean nsxOwned) { + Site2SiteVpnConnection connection = Mockito.mock(Site2SiteVpnConnection.class); + when(connection.getVpnGatewayId()).thenReturn(7L); + Mockito.lenient().when(connection.getCustomerGatewayId()).thenReturn(3L); + Site2SiteVpnGatewayVO vpnGateway = Mockito.mock(Site2SiteVpnGatewayVO.class); + when(vpnGatewayDao.findById(7L)).thenReturn(vpnGateway); + when(vpnGateway.getVpcId()).thenReturn(9L); + when(vpnGateway.getAddrId()).thenReturn(30L); + if (nsxOwned) { + when(userIpAddressDetailsDao.findDetail(30L, "nsxVpnGatewayIp")) + .thenReturn(Mockito.mock(UserIpAddressDetailVO.class)); + } + when(vpcDao.findById(9L)).thenReturn(vpcVO); + return connection; + } + + @Test + public void testStartSite2SiteVpn() throws ResourceUnavailableException { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO); + when(connection.getId()).thenReturn(5L); + Site2SiteVpnGatewayVO vpnGateway = vpnGatewayDao.findById(7L); + when(vpnGateway.getAddrId()).thenReturn(30L); + mockIpAddressVO(30L, "10.1.13.30"); + Site2SiteCustomerGatewayVO customerGateway = mockCustomerGateway("aes256-sha256;modp2048", "aes128-sha1"); + when(customerGateway.getGatewayIp()).thenReturn("203.0.113.10"); + when(customerGateway.getGuestCidrList()).thenReturn("192.168.100.0/24,192.168.200.0/24"); + when(nsxService.createVpnConnection(any(Vpc.class), any(), anyString(), anyString(), anyString(), anyString(), + anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyList(), + eq("169.254.64.21"), eq("169.254.64.22"), anyInt(), eq("10.1.13.30"))).thenReturn(true); + + assertTrue(nsxElement.startSite2SiteVpn(connection)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testStartSite2SiteVpnRejectsUnsupportedCrypto() throws ResourceUnavailableException { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO); + mockCustomerGateway("3des-md5;modp1024", "3des-md5"); + + nsxElement.startSite2SiteVpn(connection); + } + + @Test(expected = InvalidParameterValueException.class) + public void testStartSite2SiteVpnRejectsDnsPeerAddress() throws ResourceUnavailableException { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO); + Site2SiteCustomerGatewayVO customerGateway = mockCustomerGateway("aes256-sha256;modp2048", "aes128-sha1"); + when(customerGateway.getName()).thenReturn("remote-site"); + when(customerGateway.getGatewayIp()).thenReturn("vpn.example.test"); + + nsxElement.startSite2SiteVpn(connection); + } + + @Test + public void testStartSite2SiteVpnIsNoOpWhenVpnIsNotProvidedByNsx() throws ResourceUnavailableException { + VpcVO vpcVO = Mockito.mock(VpcVO.class); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO, false); + + assertTrue(nsxElement.startSite2SiteVpn(connection)); + verify(nsxService, Mockito.never()).createVpnConnection(any(Vpc.class), any(), anyString(), anyString(), + anyString(), anyString(), anyLong(), anyLong(), anyBoolean(), anyString(), anyBoolean(), anyList(), + anyString(), anyString(), anyInt(), anyString()); + } + + @Test(expected = CloudRuntimeException.class) + public void testStartSite2SiteVpnThrowsWhenVpcIsMissing() throws ResourceUnavailableException { + Site2SiteVpnConnection connection = mockVpnConnection(null); + + nsxElement.startSite2SiteVpn(connection); + } + + @Test + public void testStopSite2SiteVpn() throws ResourceUnavailableException { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO); + when(connection.getUuid()).thenReturn("conn-uuid"); + when(nsxService.updateVpnConnectionState(vpcVO, "conn-uuid", false)).thenReturn(true); + + assertTrue(nsxElement.stopSite2SiteVpn(connection)); + } + + @Test(expected = CloudRuntimeException.class) + public void testStopSite2SiteVpnThrowsWhenVpcIsMissing() throws ResourceUnavailableException { + Site2SiteVpnConnection connection = mockVpnConnection(null); + + nsxElement.stopSite2SiteVpn(connection); + } + + @Test + public void testDeleteSite2SiteVpnRemovesTheProviderConnection() throws ResourceUnavailableException { + VpcVO vpcVO = mockVpcWithNsxVpnSupport(); + Site2SiteVpnConnection connection = mockVpnConnection(vpcVO); + when(connection.getUuid()).thenReturn("conn-uuid"); + when(nsxService.deleteVpnConnection(vpcVO, "conn-uuid")).thenReturn(true); + + assertTrue(nsxElement.deleteSite2SiteVpn(connection)); + } + + @Test(expected = CloudRuntimeException.class) + public void testStopSite2SiteVpnThrowsWhenVpnGatewayIsMissing() throws ResourceUnavailableException { + Site2SiteVpnConnection connection = Mockito.mock(Site2SiteVpnConnection.class); + when(connection.getVpnGatewayId()).thenReturn(7L); + when(vpnGatewayDao.findById(7L)).thenReturn(null); + + nsxElement.stopSite2SiteVpn(connection); + } } diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java index cb79873f364d..9a41d20bd320 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxGuestNetworkGuruTest.java @@ -34,6 +34,7 @@ import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.guru.GuestNetworkGuru; +import com.cloud.network.nsx.NsxService; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.offering.NetworkOffering; @@ -51,13 +52,13 @@ import org.apache.cloudstack.NsxAnswer; import org.apache.cloudstack.agent.api.CreateNsxDhcpRelayConfigCommand; import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; -import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.NsxCommand; import org.apache.cloudstack.utils.NsxControllerUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; @@ -66,6 +67,7 @@ import org.springframework.test.util.ReflectionTestUtils; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; @@ -120,6 +122,8 @@ public class NsxGuestNetworkGuruTest { IpAddressManager ipAddressManager; @Mock NetworkOfferingDao networkOfferingDao; + @Mock + NsxService nsxService; NsxGuestNetworkGuru guru; AutoCloseable closeable; @@ -140,6 +144,7 @@ public void setUp() throws IllegalAccessException, NoSuchFieldException { guru.networkOfferingServiceMapDao = networkOfferingServiceMapDao; guru.nsxControllerUtils = nsxControllerUtils; + guru.nsxService = nsxService; guru.accountDao = accountDao; guru.domainDao = domainDao; @@ -304,6 +309,29 @@ public void testCreateNsxSegmentForVpc() { anyLong()); } + @Test + public void testCreateNsxSegmentPassesNetworkOfferingProfiles() { + NetworkVO networkVO = Mockito.mock(NetworkVO.class); + DataCenter dataCenter = Mockito.mock(DataCenter.class); + when(networkVO.getAccountId()).thenReturn(1L); + when(networkVO.getNetworkOfferingId()).thenReturn(42L); + when(networkModel.getNtwkOffDetails(42L)).thenReturn(Map.of( + NetworkOffering.Detail.NsxIpDiscoveryProfileId, "ip-profile", + NetworkOffering.Detail.NsxMacDiscoveryProfileId, "mac-profile", + NetworkOffering.Detail.NsxSegmentSecurityProfileId, "security-profile")); + when(nsxControllerUtils.sendNsxCommand(any(CreateNsxSegmentCommand.class), anyLong())) + .thenReturn(new NsxAnswer(new NsxCommand(), true, "")); + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(CreateNsxSegmentCommand.class); + + guru.createNsxSegment(networkVO, dataCenter); + + verify(nsxControllerUtils).sendNsxCommand(commandCaptor.capture(), anyLong()); + CreateNsxSegmentCommand command = commandCaptor.getValue(); + assertEquals("ip-profile", command.getIpDiscoveryProfileId()); + assertEquals("mac-profile", command.getMacDiscoveryProfileId()); + assertEquals("security-profile", command.getSegmentSecurityProfileId()); + } + @Test public void testCreateNsxSegmentForIsolatedNetwork() { @@ -313,16 +341,17 @@ public void testCreateNsxSegmentForIsolatedNetwork() { when(networkVO.getAccountId()).thenReturn(1L); when(networkVO.getVpcId()).thenReturn(null); - when(nsxControllerUtils.sendNsxCommand(any(CreateNsxTier1GatewayCommand.class), - anyLong())).thenReturn(new NsxAnswer(new NsxCommand(), true, "")); + when(nsxService.createNetwork(anyLong(), anyLong(), anyLong(), anyLong(), any(), + ArgumentMatchers.anyBoolean())).thenReturn(true); when(nsxControllerUtils.sendNsxCommand(any(CreateNsxSegmentCommand.class), anyLong())).thenReturn(new NsxAnswer(new NsxCommand(), true, "")); when(networkVO.getNetworkOfferingId()).thenReturn(1L); when(networkOfferingDao.findById(1L)).thenReturn(offeringVO); when(offeringVO.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.NATTED); + when(networkOfferingServiceMapDao.areServicesSupportedByNetworkOffering(1L, Network.Service.SourceNat)).thenReturn(true); guru.createNsxSegment(networkVO, dataCenter); - verify(nsxControllerUtils, times(1)).sendNsxCommand(any(CreateNsxTier1GatewayCommand.class), - anyLong()); + verify(nsxService).createNetwork(anyLong(), anyLong(), anyLong(), anyLong(), any(), + ArgumentMatchers.eq(true)); verify(nsxControllerUtils, times(1)).sendNsxCommand(any(CreateNsxSegmentCommand.class), anyLong()); } diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxProviderServiceImplTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxProviderServiceImplTest.java index cb6f6511d24d..ed5e72e08608 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxProviderServiceImplTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxProviderServiceImplTest.java @@ -17,15 +17,27 @@ package org.apache.cloudstack.service; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.AccountVlanMapVO; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanDetailsVO; +import com.cloud.dc.VlanVO; +import com.cloud.dc.dao.AccountVlanMapDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.DomainVlanMapDao; +import com.cloud.dc.dao.VlanDao; +import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.host.Host; import com.cloud.host.dao.HostDetailsDao; import com.cloud.network.Network; +import com.cloud.network.IpAddress; import com.cloud.network.Networks; import com.cloud.network.nsx.NsxProvider; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.element.NsxProviderVO; @@ -33,19 +45,36 @@ import com.cloud.resource.ServerResource; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.BaseResponse; +import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.api.command.AddNsxControllerCmd; +import org.apache.cloudstack.api.command.AddNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.AssignNsxVrfGatewayCmd; +import org.apache.cloudstack.api.command.ListNsxVrfGatewaysCmd; +import org.apache.cloudstack.api.response.NsxVrfGatewayResponse; import org.apache.cloudstack.api.response.NsxControllerResponse; +import org.apache.cloudstack.NsxVrfGatewayValidationAnswer; +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; +import org.apache.cloudstack.utils.NsxControllerUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import java.lang.reflect.InvocationTargetException; import java.util.List; import java.util.UUID; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -53,8 +82,11 @@ import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -71,6 +103,32 @@ public class NsxProviderServiceImplTest { ResourceManager resourceManager; @Mock HostDetailsDao hostDetailsDao; + @Mock + NsxVrfGatewayDao nsxVrfGatewayDao; + @Mock + NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Mock + IPAddressDao ipAddressDao; + @Mock + AccountDao accountDao; + @Mock + DomainDao domainDao; + @Mock + VpcDao vpcDao; + @Mock + VpcOfferingServiceMapDao vpcOfferingServiceMapDao; + @Mock + VlanDao vlanDao; + @Mock + VlanDetailsDao vlanDetailsDao; + @Mock + AccountVlanMapDao accountVlanMapDao; + @Mock + DomainVlanMapDao domainVlanMapDao; + @Mock + NsxControllerUtils nsxControllerUtils; + @Mock + NsxVrfGatewayLockManager nsxVrfGatewayLockManager; NsxProviderServiceImpl nsxProviderService; @@ -83,6 +141,26 @@ public void setup() { nsxProviderService.dataCenterDao = dataCenterDao; nsxProviderService.networkDao = networkDao; nsxProviderService.physicalNetworkDao = physicalNetworkDao; + nsxProviderService.nsxVrfGatewayDao = nsxVrfGatewayDao; + nsxProviderService.nsxVrfGatewayPlacementDao = nsxVrfGatewayPlacementDao; + nsxProviderService.ipAddressDao = ipAddressDao; + nsxProviderService.accountDao = accountDao; + nsxProviderService.domainDao = domainDao; + nsxProviderService.vpcDao = vpcDao; + nsxProviderService.vpcOfferingServiceMapDao = vpcOfferingServiceMapDao; + nsxProviderService.vlanDao = vlanDao; + nsxProviderService.vlanDetailsDao = vlanDetailsDao; + nsxProviderService.accountVlanMapDao = accountVlanMapDao; + nsxProviderService.domainVlanMapDao = domainVlanMapDao; + nsxProviderService.nsxControllerUtils = nsxControllerUtils; + nsxProviderService.nsxVrfGatewayLockManager = nsxVrfGatewayLockManager; + Mockito.lenient().when(nsxVrfGatewayLockManager.withZoneLock(anyLong(), any())) + .thenAnswer(invocation -> ((Supplier) invocation.getArgument(1)).get()); + Mockito.lenient().when(nsxVrfGatewayPlacementDao.listByZone(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(vpcDao.listByZone(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(networkDao.listByZone(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(accountVlanMapDao.listAccountVlanMapsByVlan(anyLong())).thenReturn(List.of()); + Mockito.lenient().when(domainVlanMapDao.listDomainVlanMapsByVlan(anyLong())).thenReturn(List.of()); } @Test @@ -171,4 +249,370 @@ public void testNetworkStateValidation() throws NoSuchMethodException, Invocatio assertThrows(CloudRuntimeException.class, () -> nsxProviderService.validateNetworkState(networkVOList)); } + + private static final long VRF_ZONE_ID = 1L; + private static final long VRF_PUBLIC_VLAN_ID = 71L; + + private NsxVrfGatewayVO stagedGateway() { + NsxVrfGatewayVO gateway = + new NsxVrfGatewayVO(VRF_ZONE_ID, "CS-VRF-001", "v5-EdgeCluster-HOSTED", "v5-T0"); + gateway.setPublicVlanDbId(VRF_PUBLIC_VLAN_ID); + return gateway; + } + + private AddNsxVrfGatewayCmd addCmd() { + AddNsxVrfGatewayCmd cmd = mock(AddNsxVrfGatewayCmd.class); + when(cmd.getZoneId()).thenReturn(VRF_ZONE_ID); + when(cmd.getTier0Gateway()).thenReturn("CS-VRF-001"); + when(cmd.getParentTier0Gateway()).thenReturn("v5-T0"); + when(cmd.getEdgeCluster()).thenReturn("v5-EdgeCluster-HOSTED"); + when(cmd.getPublicVlanId()).thenReturn(VRF_PUBLIC_VLAN_ID); + return cmd; + } + + private DataCenterVO mockZoneAndProvider() { + DataCenterVO zone = mock(DataCenterVO.class); + when(zone.getId()).thenReturn(VRF_ZONE_ID); + when(dataCenterDao.findById(VRF_ZONE_ID)).thenReturn(zone); + NsxProviderVO provider = mock(NsxProviderVO.class); + when(provider.getTier0Gateway()).thenReturn("v5-T0"); + when(nsxProviderDao.findByZoneId(VRF_ZONE_ID)).thenReturn(provider); + return zone; + } + + private VlanVO mockValidPublicVlan() { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(VRF_PUBLIC_VLAN_ID); + when(vlan.getDataCenterId()).thenReturn(VRF_ZONE_ID); + when(vlan.getVlanType()).thenReturn(Vlan.VlanType.VirtualNetwork); + when(vlan.getUuid()).thenReturn("public-vlan-uuid"); + when(vlanDao.findById(VRF_PUBLIC_VLAN_ID)).thenReturn(vlan); + VlanDetailsVO detail = mock(VlanDetailsVO.class); + when(detail.getValue()).thenReturn(Boolean.TRUE.toString()); + when(vlanDetailsDao.findDetail(eq(VRF_PUBLIC_VLAN_ID), any())).thenReturn(detail); + when(ipAddressDao.listByVlanId(VRF_PUBLIC_VLAN_ID)).thenReturn(List.of()); + return vlan; + } + + private void mockSuccessfulBackendValidation() { + NsxVrfGatewayValidationAnswer answer = mock(NsxVrfGatewayValidationAnswer.class); + when(answer.getResult()).thenReturn(true); + when(answer.getEdgeClusterPath()).thenReturn("v5-EdgeCluster-HOSTED"); + when(answer.getParentTier0Path()).thenReturn("v5-T0"); + when(nsxControllerUtils.sendNsxCommandForResult(any(ValidateNsxVrfGatewayCommand.class), + eq(VRF_ZONE_ID))).thenReturn(answer); + } + + private void mockAccountRangeDedication(long accountId) { + AccountVlanMapVO mapping = mock(AccountVlanMapVO.class); + when(mapping.getAccountId()).thenReturn(accountId); + when(accountVlanMapDao.listAccountVlanMapsByVlan(VRF_PUBLIC_VLAN_ID)).thenReturn(List.of(mapping)); + when(domainVlanMapDao.listDomainVlanMapsByVlan(VRF_PUBLIC_VLAN_ID)).thenReturn(List.of()); + } + + @Test + public void testAddNsxVrfGatewayPersistsTheRegistration() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + when(nsxVrfGatewayDao.findByZoneAndTier0Name(VRF_ZONE_ID, "CS-VRF-001")).thenReturn(null); + when(nsxVrfGatewayDao.persist(any(NsxVrfGatewayVO.class))).thenAnswer(i -> i.getArgument(0)); + ArgumentCaptor gatewayCaptor = ArgumentCaptor.forClass(NsxVrfGatewayVO.class); + + AddNsxVrfGatewayCmd cmd = addCmd(); + + NsxVrfGatewayResponse response = nsxProviderService.addNsxVrfGateway(cmd); + + assertEquals("CS-VRF-001", response.getTier0Gateway()); + assertEquals("v5-EdgeCluster-HOSTED", response.getEdgeCluster()); + Assert.assertFalse(response.isAllocated()); + verify(ipAddressDao).lockRange(VRF_PUBLIC_VLAN_ID); + verify(nsxVrfGatewayDao).persist(gatewayCaptor.capture()); + assertEquals(Long.valueOf(VRF_PUBLIC_VLAN_ID), gatewayCaptor.getValue().getPublicVlanDbId()); + } + + @Test + public void testAddNsxVrfGatewayRejectsDuplicateTier0InTheSameZone() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + when(nsxVrfGatewayDao.findByZoneAndTier0Name(VRF_ZONE_ID, "CS-VRF-001")).thenReturn(stagedGateway()); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.addNsxVrfGateway(addCmd())); + } + + @Test + public void testAddNsxVrfGatewayRejectsPublicRangeReservedForSystemVms() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + IPAddressVO systemIp = mock(IPAddressVO.class); + when(systemIp.isForSystemVms()).thenReturn(true); + when(ipAddressDao.listByVlanId(VRF_PUBLIC_VLAN_ID)).thenReturn(List.of(systemIp)); + + assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.addNsxVrfGateway(addCmd())); + + verify(nsxVrfGatewayDao, never()).persist(any(NsxVrfGatewayVO.class)); + } + + @Test + public void testAddNsxVrfGatewayRejectsPublicRangeWithAllocatedAddresses() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + IPAddressVO allocatedIp = mock(IPAddressVO.class); + when(allocatedIp.getState()).thenReturn(IpAddress.State.Allocated); + when(ipAddressDao.listByVlanId(VRF_PUBLIC_VLAN_ID)).thenReturn(List.of(allocatedIp)); + + assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.addNsxVrfGateway(addCmd())); + + verify(nsxVrfGatewayDao, never()).persist(any(NsxVrfGatewayVO.class)); + } + + @Test + public void testAddNsxVrfGatewayRejectsPublicRangeAlreadyRegisteredElsewhere() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + when(nsxVrfGatewayDao.findByPublicVlan(VRF_PUBLIC_VLAN_ID)).thenReturn(stagedGateway()); + + assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.addNsxVrfGateway(addCmd())); + + verify(nsxVrfGatewayDao, never()).persist(any(NsxVrfGatewayVO.class)); + } + + @Test + public void testAddNsxVrfGatewayRejectsZoneWithoutNsxController() { + DataCenterVO zone = mock(DataCenterVO.class); + when(zone.getId()).thenReturn(VRF_ZONE_ID); + when(dataCenterDao.findById(VRF_ZONE_ID)).thenReturn(zone); + when(nsxProviderDao.findByZoneId(VRF_ZONE_ID)).thenReturn(null); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.addNsxVrfGateway(addCmd())); + } + + @Test + public void testAddNsxVrfGatewayRejectsMissingBackendAnswerCleanly() { + mockZoneAndProvider(); + when(nsxControllerUtils.sendNsxCommandForResult(any(ValidateNsxVrfGatewayCommand.class), + eq(VRF_ZONE_ID))).thenReturn(null); + + InvalidParameterValueException exception = assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.addNsxVrfGateway(addCmd())); + + assertTrue(exception.getMessage(), exception.getMessage().contains("no answer was returned")); + verify(nsxVrfGatewayDao, never()).persist(any(NsxVrfGatewayVO.class)); + } + + @Test + public void testAddNsxVrfGatewayFailsWhenRegistrationIsNotPersisted() { + mockZoneAndProvider(); + mockValidPublicVlan(); + mockSuccessfulBackendValidation(); + when(nsxVrfGatewayDao.persist(any(NsxVrfGatewayVO.class))).thenReturn(null); + + assertThrows(CloudRuntimeException.class, () -> nsxProviderService.addNsxVrfGateway(addCmd())); + } + + @Test + public void testAssignNsxVrfGatewayToAccount() { + NsxVrfGatewayVO gateway = stagedGateway(); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + mockAccountRangeDedication(9L); + AccountVO account = mock(AccountVO.class); + when(accountDao.findById(9L)).thenReturn(account); + when(nsxVrfGatewayDao.findByAccount(VRF_ZONE_ID, 9L)).thenReturn(null); + when(nsxVrfGatewayDao.update(anyLong(), any(NsxVrfGatewayVO.class))).thenReturn(true); + + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(null); + when(cmd.getDomainId()).thenReturn(null); + + NsxVrfGatewayResponse response = nsxProviderService.assignNsxVrfGateway(cmd); + + assertEquals(NsxVrfGatewayVO.Scope.ACCOUNT.name(), gateway.getScope()); + assertEquals(Long.valueOf(9L), gateway.getAccountId()); + assertTrue(response.isAllocated()); + } + + @Test + public void testAssignNsxVrfGatewayRequiresExactlyOneOfAccountOrDomain() { + when(nsxVrfGatewayDao.findById(5L)).thenReturn(stagedGateway()); + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(2L); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.assignNsxVrfGateway(cmd)); + } + + @Test + public void testAssignNsxVrfGatewayRejectsMissingAccountBeforeRangeChecks() { + when(nsxVrfGatewayDao.findById(5L)).thenReturn(stagedGateway()); + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(null); + + InvalidParameterValueException exception = assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.assignNsxVrfGateway(cmd)); + + assertTrue(exception.getMessage(), exception.getMessage().contains("Could not find account")); + verify(accountVlanMapDao, never()).listAccountVlanMapsByVlan(anyLong()); + } + + @Test + public void testAssignNsxVrfGatewayRejectsAlreadyAssignedGateway() { + NsxVrfGatewayVO gateway = stagedGateway(); + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(3L); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(null); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.assignNsxVrfGateway(cmd)); + } + + @Test + public void testAssignNsxVrfGatewayRejectsExistingTenantPlacement() { + NsxVrfGatewayVO gateway = stagedGateway(); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(accountDao.findById(9L)).thenReturn(mock(AccountVO.class)); + com.cloud.network.element.NsxVrfGatewayPlacementVO placement = + mock(com.cloud.network.element.NsxVrfGatewayPlacementVO.class); + when(placement.getAccountId()).thenReturn(9L); + when(nsxVrfGatewayPlacementDao.listByZone(VRF_ZONE_ID)).thenReturn(List.of(placement)); + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(null); + + InvalidParameterValueException exception = assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.assignNsxVrfGateway(cmd)); + + assertTrue(exception.getMessage(), exception.getMessage().contains("already has an NSX Tier-1 placement")); + verify(nsxVrfGatewayDao, never()).update(anyLong(), any(NsxVrfGatewayVO.class)); + } + + @Test + public void testAssignNsxVrfGatewayRejectsLegacyNsxNetworkWithoutPlacement() { + NsxVrfGatewayVO gateway = stagedGateway(); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(accountDao.findById(9L)).thenReturn(mock(AccountVO.class)); + NetworkVO network = mock(NetworkVO.class); + when(network.getId()).thenReturn(81L); + when(network.getVpcId()).thenReturn(null); + when(network.getBroadcastDomainType()).thenReturn(Networks.BroadcastDomainType.NSX); + when(network.getAccountId()).thenReturn(9L); + when(networkDao.listByZone(VRF_ZONE_ID)).thenReturn(List.of(network)); + when(nsxVrfGatewayPlacementDao.findByNetworkId(81L)).thenReturn(null); + AssignNsxVrfGatewayCmd cmd = mock(AssignNsxVrfGatewayCmd.class); + when(cmd.getId()).thenReturn(5L); + when(cmd.getAccountId()).thenReturn(9L); + when(cmd.getDomainId()).thenReturn(null); + + InvalidParameterValueException exception = assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.assignNsxVrfGateway(cmd)); + + assertTrue(exception.getMessage(), exception.getMessage().contains("without recorded placement")); + verify(nsxVrfGatewayDao, never()).update(anyLong(), any(NsxVrfGatewayVO.class)); + } + + @Test + public void testReleaseNsxVrfGatewayRefusesWhileNetworksAreAttached() { + NsxVrfGatewayVO gateway = stagedGateway(); + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(9L); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId())).thenReturn(1L); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.releaseNsxVrfGateway(5L)); + } + + @Test + public void testReleaseNsxVrfGatewayReturnsItToThePool() { + NsxVrfGatewayVO gateway = stagedGateway(); + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(9L); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId())).thenReturn(0L); + when(nsxVrfGatewayDao.update(anyLong(), any(NsxVrfGatewayVO.class))).thenReturn(true); + + NsxVrfGatewayResponse response = nsxProviderService.releaseNsxVrfGateway(5L); + + assertTrue(gateway.isUnclaimed()); + Assert.assertFalse(response.isAllocated()); + verify(nsxVrfGatewayDao).update(eq(gateway.getId()), eq(gateway)); + } + + /** + * Regression test for a release that reported success while persisting nothing. + * + * Entities are CGLIB-enhanced and GenericDaoBase builds its UPDATE from setter calls + * intercepted by UpdateBuilder. Clearing the fields directly on the VO left the row + * untouched in the database, so the gateway stayed assigned and could never be + * deregistered. Asserting on the returned object alone did not catch it, because that + * object is the in-memory VO; the DAO contract is what matters. + */ + @Test + public void testReleaseNsxVrfGatewayFailsLoudlyWhenTheUpdateDoesNotPersist() { + NsxVrfGatewayVO gateway = stagedGateway(); + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(9L); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId())).thenReturn(0L); + when(nsxVrfGatewayDao.update(anyLong(), any(NsxVrfGatewayVO.class))).thenReturn(false); + + assertThrows(CloudRuntimeException.class, () -> nsxProviderService.releaseNsxVrfGateway(5L)); + } + + @Test + public void testDeleteNsxVrfGatewayRefusesWhileAssigned() { + NsxVrfGatewayVO gateway = stagedGateway(); + gateway.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + gateway.setAccountId(9L); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + + assertThrows(InvalidParameterValueException.class, () -> nsxProviderService.deleteNsxVrfGateway(5L)); + } + + @Test + public void testDeleteNsxVrfGatewayRefusesWhilePlacementExists() { + NsxVrfGatewayVO gateway = stagedGateway(); + when(nsxVrfGatewayDao.findById(5L)).thenReturn(gateway); + when(nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId())).thenReturn(1L); + + assertThrows(InvalidParameterValueException.class, + () -> nsxProviderService.deleteNsxVrfGateway(5L)); + + verify(nsxVrfGatewayDao, never()).expunge(anyLong()); + } + + @Test + public void testListNsxVrfGatewaysFiltersByAllocation() { + NsxVrfGatewayVO unclaimed = stagedGateway(); + NsxVrfGatewayVO claimed = new NsxVrfGatewayVO(VRF_ZONE_ID, "CS-VRF-002", "v5-EdgeCluster-HOSTED", "v5-T0"); + claimed.setScope(NsxVrfGatewayVO.Scope.ACCOUNT.name()); + claimed.setAccountId(9L); + when(nsxVrfGatewayDao.listByZone(VRF_ZONE_ID)).thenReturn(List.of(unclaimed, claimed)); + + ListNsxVrfGatewaysCmd cmd = mock(ListNsxVrfGatewaysCmd.class); + when(cmd.getZoneId()).thenReturn(VRF_ZONE_ID); + when(cmd.getAccountId()).thenReturn(null); + when(cmd.getDomainId()).thenReturn(null); + when(cmd.getAllocatedOnly()).thenReturn(Boolean.TRUE); + + List responses = nsxProviderService.listNsxVrfGateways(cmd); + + assertEquals(1, responses.size()); + assertEquals("CS-VRF-002", responses.get(0).getTier0Gateway()); + } } diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxPublicNetworkGuruTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxPublicNetworkGuruTest.java index ae3f3f3ecd55..e5a635467dd2 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxPublicNetworkGuruTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxPublicNetworkGuruTest.java @@ -167,7 +167,7 @@ public void testAllocate() throws InsufficientVirtualNetworkCapacityException, I when(vpcVO.getId()).thenReturn(12L); when(vpcVO.getName()).thenReturn("nsxVPCNet"); when(vpcOfferingServiceMapDao.areServicesSupportedByVpcOffering(anyLong(), any())).thenReturn(true); - when(nsxService.createVpcNetwork(anyLong(), anyLong(), anyLong(), anyLong(), anyString(), anyBoolean())).thenReturn(true); + when(nsxService.createVpcNetwork(anyLong(), anyLong(), anyLong(), anyLong(), anyString(), anyBoolean(), anyLong())).thenReturn(true); when(vpcOfferingDao.findById(anyLong())).thenReturn(vpcOffering); when(vpcOffering.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.NATTED); when(nsxControllerUtils.sendNsxCommand(any(CreateOrUpdateNsxTier1NatRuleCommand.class), diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxServiceImplTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxServiceImplTest.java index 41f47bc610e5..9e97d1faa807 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxServiceImplTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxServiceImplTest.java @@ -17,32 +17,71 @@ package org.apache.cloudstack.service; import com.cloud.network.IpAddress; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.element.NsxVrfGatewayPlacementVO; +import com.cloud.network.element.NsxProviderVO; +import com.cloud.network.Site2SiteVpnConnection; +import com.cloud.network.dao.Site2SiteVpnConnectionVO; +import com.cloud.network.dao.Site2SiteVpnConnectionDao; +import com.cloud.network.dao.Site2SiteVpnGatewayDao; +import com.cloud.network.dao.Site2SiteVpnGatewayVO; +import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.network.nsx.NsxVpnGatewayResult; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import org.apache.cloudstack.NsxAnswer; +import org.apache.cloudstack.NsxVrfGatewayValidationAnswer; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.CreateNsxVpnGatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; +import org.apache.cloudstack.agent.api.ValidateNsxVrfGatewayCommand; import org.apache.cloudstack.utils.NsxControllerUtils; +import org.apache.cloudstack.resourcedetail.UserIpAddressDetailVO; +import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnitRunner; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -51,6 +90,22 @@ public class NsxServiceImplTest { private NsxControllerUtils nsxControllerUtils; @Mock private VpcDao vpcDao; + @Mock + private NsxVrfGatewayDao nsxVrfGatewayDao; + @Mock + private NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Mock + private NsxProviderDao nsxProviderDao; + @Mock + private NsxVrfGatewayLockManager nsxVrfGatewayLockManager; + @Mock + private DomainDao domainDao; + @Mock + private Site2SiteVpnConnectionDao site2SiteVpnConnectionDao; + @Mock + private Site2SiteVpnGatewayDao site2SiteVpnGatewayDao; + @Mock + private UserIpAddressDetailsDao userIpAddressDetailsDao; NsxServiceImpl nsxService; AutoCloseable closeable; @@ -62,9 +117,31 @@ public class NsxServiceImplTest { @Before public void setup() { closeable = MockitoAnnotations.openMocks(this); - nsxService = new NsxServiceImpl(); + nsxService = Mockito.spy(new NsxServiceImpl()); nsxService.nsxControllerUtils = nsxControllerUtils; nsxService.vpcDao = vpcDao; + nsxService.nsxVrfGatewayDao = nsxVrfGatewayDao; + nsxService.nsxVrfGatewayPlacementDao = nsxVrfGatewayPlacementDao; + nsxService.nsxProviderDao = nsxProviderDao; + nsxService.nsxVrfGatewayLockManager = nsxVrfGatewayLockManager; + nsxService.domainDao = domainDao; + nsxService.site2SiteVpnConnectionDao = site2SiteVpnConnectionDao; + nsxService.site2SiteVpnGatewayDao = site2SiteVpnGatewayDao; + nsxService.userIpAddressDetailsDao = userIpAddressDetailsDao; + Mockito.lenient().when(nsxVrfGatewayLockManager.withPlacementLock(anyBoolean(), anyLong(), any())) + .thenAnswer(invocation -> ((Supplier) invocation.getArgument(2)).get()); + Mockito.lenient().when(nsxVrfGatewayLockManager.withZoneLock(anyLong(), any())) + .thenAnswer(invocation -> ((Supplier) invocation.getArgument(1)).get()); + Mockito.lenient().when(nsxVrfGatewayPlacementDao.persist(any(NsxVrfGatewayPlacementVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.lenient().when(nsxVrfGatewayPlacementDao.update(anyLong(), any(NsxVrfGatewayPlacementVO.class))).thenReturn(true); + NsxProviderVO provider = mock(NsxProviderVO.class); + Mockito.lenient().when(provider.getTier0Gateway()).thenReturn("v5-T0"); + Mockito.lenient().when(nsxProviderDao.findByZoneId(anyLong())).thenReturn(provider); + NsxVrfGatewayValidationAnswer validationAnswer = mock(NsxVrfGatewayValidationAnswer.class); + Mockito.lenient().when(validationAnswer.getResult()).thenReturn(true); + Mockito.lenient().when(nsxControllerUtils.sendNsxCommandForResult(any(ValidateNsxVrfGatewayCommand.class), anyLong())) + .thenReturn(validationAnswer); } @After @@ -72,13 +149,243 @@ public void teardown() throws Exception { closeable.close(); } + private NsxVrfGatewayVO vrfGateway(String tier0, String edgeCluster) { + return new NsxVrfGatewayVO(zoneId, tier0, edgeCluster, "v5-T0"); + } + + private NsxVrfGatewayPlacementVO placement(Long gatewayId, Long vpcId, Long networkId, + String tier0, NsxVrfGatewayPlacementVO.State state) { + NsxVrfGatewayPlacementVO placement = new NsxVrfGatewayPlacementVO(gatewayId, zoneId, domainId, + accountId, vpcId, networkId, tier0); + placement.setState(state); + return placement; + } + + @Test + public void testResolveVrfGatewayReturnsNullWhenScopeIsNone() { + Mockito.doReturn("NONE").when(nsxService).getVrfScope(zoneId); + assertNull(nsxService.resolveVrfGateway(zoneId, accountId, domainId)); + verify(nsxVrfGatewayDao, never()).findByAccount(anyLong(), anyLong()); + } + + @Test + public void testResolveVrfGatewayReturnsNullWhenZoneIsNull() { + assertNull(nsxService.resolveVrfGateway(null, accountId, domainId)); + } + + @Test + public void testResolveVrfGatewayPrefersTheAccountAssignment() { + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + NsxVrfGatewayVO expected = vrfGateway("CS-VRF-001", "v5-EdgeCluster-HOSTED"); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)).thenReturn(expected); + assertEquals(expected, nsxService.resolveVrfGateway(zoneId, accountId, domainId)); + verify(domainDao, never()).findById(anyLong()); + } + + @Test + public void testResolveVrfGatewayFallsBackToTheDomainWhenTheAccountHasNone() { + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + NsxVrfGatewayVO expected = vrfGateway("CS-VRF-002", "v5-EdgeCluster-HOSTED"); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)).thenReturn(null); + DomainVO domain = mock(DomainVO.class); + when(domain.getId()).thenReturn(domainId); + when(domainDao.findById(domainId)).thenReturn(domain); + when(nsxVrfGatewayDao.findByDomain(zoneId, domainId)).thenReturn(expected); + assertEquals(expected, nsxService.resolveVrfGateway(zoneId, accountId, domainId)); + } + + @Test + public void testResolveVrfGatewayWalksUpTheDomainChain() { + Mockito.doReturn("DOMAIN").when(nsxService).getVrfScope(zoneId); + long parentDomainId = 7L; + NsxVrfGatewayVO expected = vrfGateway("CS-VRF-003", "v5-EdgeCluster-HOSTED"); + DomainVO child = mock(DomainVO.class); + when(child.getId()).thenReturn(domainId); + when(child.getParent()).thenReturn(parentDomainId); + DomainVO parent = mock(DomainVO.class); + when(parent.getId()).thenReturn(parentDomainId); + when(domainDao.findById(domainId)).thenReturn(child); + when(domainDao.findById(parentDomainId)).thenReturn(parent); + when(nsxVrfGatewayDao.findByDomain(zoneId, domainId)).thenReturn(null); + when(nsxVrfGatewayDao.findByDomain(zoneId, parentDomainId)).thenReturn(expected); + assertEquals(expected, nsxService.resolveVrfGateway(zoneId, accountId, domainId)); + } + + @Test + public void testResolveVrfGatewayFallsBackToSharedTier0WhenAllowed() { + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + Mockito.doReturn(true).when(nsxService).isVrfFallbackToSharedTier0Allowed(zoneId); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)).thenReturn(null); + when(domainDao.findById(domainId)).thenReturn(null); + assertNull(nsxService.resolveVrfGateway(zoneId, accountId, domainId)); + } + + @Test(expected = CloudRuntimeException.class) + public void testResolveVrfGatewayFailsWhenSegregatedAndUnassigned() { + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + Mockito.doReturn(false).when(nsxService).isVrfFallbackToSharedTier0Allowed(zoneId); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)).thenReturn(null); + when(domainDao.findById(domainId)).thenReturn(null); + nsxService.resolveVrfGateway(zoneId, accountId, domainId); + } + @Test public void testCreateVpcNetwork() { NsxAnswer createNsxTier1GatewayAnswer = mock(NsxAnswer.class); when(nsxControllerUtils.sendNsxCommand(any(CreateNsxTier1GatewayCommand.class), anyLong())).thenReturn(createNsxTier1GatewayAnswer); when(createNsxTier1GatewayAnswer.getResult()).thenReturn(true); - assertTrue(nsxService.createVpcNetwork(1L, 3L, 2L, 5L, "VPC01", false)); + assertTrue(nsxService.createVpcNetwork(1L, 3L, 2L, 5L, "VPC01", false, null)); + } + + @Test + public void testCreateNetworkAppliesVrfPlacementAndSourceNatSetting() { + NsxAnswer answer = mock(NsxAnswer.class); + when(answer.getResult()).thenReturn(true); + when(nsxControllerUtils.sendNsxCommand(any(CreateNsxTier1GatewayCommand.class), eq(zoneId))) + .thenReturn(answer); + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)) + .thenReturn(vrfGateway("CS-VRF-001", "v5-EdgeCluster-HOSTED")); + ArgumentCaptor commandCaptor = + ArgumentCaptor.forClass(CreateNsxTier1GatewayCommand.class); + + assertTrue(nsxService.createNetwork(zoneId, accountId, domainId, 5L, "Network01", true)); + + verify(nsxControllerUtils).sendNsxCommand(commandCaptor.capture(), eq(zoneId)); + CreateNsxTier1GatewayCommand command = commandCaptor.getValue(); + assertEquals("CS-VRF-001", command.getTier0Gateway()); + assertEquals("v5-EdgeCluster-HOSTED", command.getEdgeCluster()); + assertTrue(command.isSourceNatEnabled()); + } + + @Test + public void testCreateNetworkFailsCleanlyWhenVrfValidationReturnsNoAnswer() { + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)) + .thenReturn(vrfGateway("CS-VRF-001", "v5-EdgeCluster-HOSTED")); + when(nsxControllerUtils.sendNsxCommandForResult(any(ValidateNsxVrfGatewayCommand.class), eq(zoneId))) + .thenReturn(null); + ArgumentCaptor placementCaptor = + ArgumentCaptor.forClass(NsxVrfGatewayPlacementVO.class); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> nsxService.createNetwork(zoneId, accountId, domainId, 5L, "Network01", true)); + + assertTrue(exception.getMessage().contains("no answer was returned")); + verify(nsxVrfGatewayPlacementDao).persist(placementCaptor.capture()); + assertEquals(NsxVrfGatewayPlacementVO.State.FAILED.name(), placementCaptor.getValue().getState()); + verify(nsxControllerUtils, never()).sendNsxCommand(any(CreateNsxTier1GatewayCommand.class), anyLong()); + } + + @Test + public void testReserveTier1PlacementPersistsResolvedVrfBeforeReturningItsVlan() { + long vpcId = 41L; + long publicVlanId = 73L; + NsxVrfGatewayVO gateway = vrfGateway("CS-VRF-001", "v5-EdgeCluster-HOSTED"); + gateway.setPublicVlanDbId(publicVlanId); + Mockito.doReturn("ACCOUNT").when(nsxService).getVrfScope(zoneId); + when(nsxVrfGatewayDao.findByAccount(zoneId, accountId)).thenReturn(gateway); + ArgumentCaptor placementCaptor = + ArgumentCaptor.forClass(NsxVrfGatewayPlacementVO.class); + + assertEquals(Long.valueOf(publicVlanId), nsxService.reserveTier1PlacementAndGetPublicVlanId( + zoneId, accountId, domainId, vpcId, null)); + + verify(nsxVrfGatewayPlacementDao).persist(placementCaptor.capture()); + NsxVrfGatewayPlacementVO persisted = placementCaptor.getValue(); + assertEquals(Long.valueOf(vpcId), persisted.getVpcId()); + assertNull(persisted.getNetworkId()); + assertEquals("CS-VRF-001", persisted.getTier0Name()); + assertEquals(NsxVrfGatewayPlacementVO.State.PENDING_CREATE.name(), persisted.getState()); + } + + @Test + public void testReserveTier1PlacementReusesExistingPlacementWithoutResolvingAgain() { + long networkId = 42L; + NsxVrfGatewayPlacementVO existing = placement(null, null, networkId, "v5-T0", + NsxVrfGatewayPlacementVO.State.ACTIVE); + when(nsxVrfGatewayPlacementDao.findByNetworkId(networkId)).thenReturn(existing); + + assertNull(nsxService.reserveTier1PlacementAndGetPublicVlanId( + zoneId, accountId, domainId, null, networkId)); + + verify(nsxVrfGatewayPlacementDao, never()).persist(any(NsxVrfGatewayPlacementVO.class)); + verify(nsxService, never()).getVrfScope(anyLong()); + } + + @Test + public void testReserveTier1PlacementAllowsRetryAfterFailedCreation() { + long vpcId = 43L; + long gatewayId = 17L; + long publicVlanId = 91L; + NsxVrfGatewayPlacementVO existing = placement(gatewayId, vpcId, null, "CS-VRF-017", + NsxVrfGatewayPlacementVO.State.FAILED); + NsxVrfGatewayVO gateway = vrfGateway("CS-VRF-017", "v5-EdgeCluster-HOSTED"); + gateway.setPublicVlanDbId(publicVlanId); + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(existing); + when(nsxVrfGatewayDao.findById(gatewayId)).thenReturn(gateway); + + assertEquals(Long.valueOf(publicVlanId), nsxService.reserveTier1PlacementAndGetPublicVlanId( + zoneId, accountId, domainId, vpcId, null)); + + verify(nsxVrfGatewayPlacementDao, never()).persist(any(NsxVrfGatewayPlacementVO.class)); + } + + @Test(expected = CloudRuntimeException.class) + public void testReadOnlyPublicVlanLookupRejectsMissingLegacyPlacement() { + long vpcId = 46L; + try { + nsxService.getPublicVlanId(zoneId, accountId, domainId, vpcId, null); + } finally { + verify(nsxVrfGatewayPlacementDao, never()).persist(any(NsxVrfGatewayPlacementVO.class)); + verify(nsxService, never()).getVrfScope(anyLong()); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testReadOnlyPublicVlanLookupRejectsFailedPlacement() { + long vpcId = 44L; + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(placement(null, vpcId, null, + "v5-T0", NsxVrfGatewayPlacementVO.State.FAILED)); + + nsxService.getPublicVlanId(zoneId, accountId, domainId, vpcId, null); + } + + @Test + public void testReadOnlyPublicVlanLookupUsesRecordedVrfGateway() { + long vpcId = 45L; + long gatewayId = 17L; + long publicVlanId = 91L; + NsxVrfGatewayPlacementVO existing = placement(gatewayId, vpcId, null, "CS-VRF-017", + NsxVrfGatewayPlacementVO.State.ACTIVE); + NsxVrfGatewayVO gateway = vrfGateway("CS-VRF-017", "v5-EdgeCluster-HOSTED"); + gateway.setPublicVlanDbId(publicVlanId); + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(existing); + when(nsxVrfGatewayDao.findById(gatewayId)).thenReturn(gateway); + + assertEquals(Long.valueOf(publicVlanId), + nsxService.getPublicVlanId(zoneId, accountId, domainId, vpcId, null)); + verify(nsxService, never()).getVrfScope(anyLong()); + } + + @Test + public void testCreateVpnGatewayPreservesStructuredFailureResult() { + Vpc vpc = mock(Vpc.class); + when(vpc.getDomainId()).thenReturn(domainId); + when(vpc.getAccountId()).thenReturn(accountId); + when(vpc.getZoneId()).thenReturn(zoneId); + when(vpc.getId()).thenReturn(3L); + NsxAnswer answer = mock(NsxAnswer.class); + when(answer.getResult()).thenReturn(false); + when(answer.isEndpointMayBeInUse()).thenReturn(true); + when(nsxControllerUtils.sendNsxCommandForResult(any(CreateNsxVpnGatewayCommand.class), eq(zoneId))) + .thenReturn(answer); + + NsxVpnGatewayResult result = nsxService.createVpnGateway(vpc, "203.0.113.20"); + + assertFalse(result.isSuccessful()); + assertTrue(result.isEndpointMayBeInUse()); } @Test @@ -90,6 +397,48 @@ public void testDeleteVpcNetwork() { assertTrue(nsxService.deleteVpcNetwork(1L, 2L, 3L, 10L, "VPC01")); } + @Test + public void testDeleteVpcNetworkRemovesPlacementAfterBackendDeletion() { + long vpcId = 10L; + NsxVrfGatewayPlacementVO existing = placement(null, vpcId, null, "v5-T0", + NsxVrfGatewayPlacementVO.State.ACTIVE); + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(existing); + when(nsxVrfGatewayPlacementDao.expunge(existing.getId())).thenReturn(true); + NsxAnswer answer = mock(NsxAnswer.class); + when(answer.getResult()).thenReturn(true); + when(nsxControllerUtils.sendNsxCommand(any(DeleteNsxTier1GatewayCommand.class), eq(zoneId))) + .thenAnswer(invocation -> { + assertEquals(NsxVrfGatewayPlacementVO.State.PENDING_DELETE.name(), existing.getState()); + return answer; + }); + + assertTrue(nsxService.deleteVpcNetwork(zoneId, accountId, domainId, vpcId, "VPC01")); + + verify(nsxVrfGatewayPlacementDao).update(existing.getId(), existing); + verify(nsxVrfGatewayPlacementDao).expunge(existing.getId()); + } + + @Test + public void testDeleteVpcNetworkMarksPlacementFailedWhenBackendDeletionFails() { + long vpcId = 10L; + NsxVrfGatewayPlacementVO existing = placement(null, vpcId, null, "v5-T0", + NsxVrfGatewayPlacementVO.State.ACTIVE); + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(existing); + NsxAnswer answer = mock(NsxAnswer.class); + when(answer.getResult()).thenReturn(false); + when(answer.getDetails()).thenReturn("backend deletion failed"); + when(nsxControllerUtils.sendNsxCommand(any(DeleteNsxTier1GatewayCommand.class), eq(zoneId))) + .thenReturn(answer); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> nsxService.deleteVpcNetwork(zoneId, accountId, domainId, vpcId, "VPC01")); + + assertTrue(exception.getMessage().contains("backend deletion failed")); + assertEquals(NsxVrfGatewayPlacementVO.State.FAILED.name(), existing.getState()); + verify(nsxVrfGatewayPlacementDao, times(2)).update(existing.getId(), existing); + verify(nsxVrfGatewayPlacementDao, never()).expunge(anyLong()); + } + @Test public void testDeleteNetworkOnVpc() { NetworkVO network = new NetworkVO(); @@ -127,6 +476,8 @@ public void testUpdateVpcSourceNatIp() { when(vpc.getDomainId()).thenReturn(domainId); when(vpc.getZoneId()).thenReturn(zoneId); when(vpc.getId()).thenReturn(vpcId); + when(nsxVrfGatewayPlacementDao.findByVpcId(vpcId)).thenReturn(placement(null, vpcId, null, + "v5-T0", NsxVrfGatewayPlacementVO.State.ACTIVE)); NsxAnswer answer = mock(NsxAnswer.class); when(answer.getResult()).thenReturn(true); when(nsxControllerUtils.sendNsxCommand(any(CreateOrUpdateNsxTier1NatRuleCommand.class), eq(zoneId))).thenReturn(answer); @@ -159,4 +510,243 @@ public void testDeleteStaticNatRule() { nsxService.deleteStaticNatRule(zoneId, domainId, accountId, networkId, networkName, true); Mockito.verify(nsxControllerUtils).sendNsxCommand(any(DeleteNsxNatRuleCommand.class), eq(zoneId)); } + + @Test + public void testPollVpnConnectionStatusTransitionsUp() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + AtomicReference transitionedState = new AtomicReference<>(); + + NsxServiceImpl service = new NsxServiceImpl() { + @Override + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + return "UP"; + } + + @Override + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + transitionedState.set(newState); + } + }; + + service.pollVpnConnectionStatus(connection, vpc); + + assertEquals(Site2SiteVpnConnection.State.Connected, transitionedState.get()); + } + + @Test + public void testPollVpnConnectionStatusTransitionsDown() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + when(connection.getState()).thenReturn(Site2SiteVpnConnection.State.Connected); + AtomicReference transitionedState = new AtomicReference<>(); + + NsxServiceImpl service = new NsxServiceImpl() { + @Override + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + return VPN_SESSION_STATUS_DOWN; + } + + @Override + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + transitionedState.set(newState); + } + }; + + service.pollVpnConnectionStatus(connection, vpc); + + assertEquals(Site2SiteVpnConnection.State.Disconnected, transitionedState.get()); + } + + @Test + public void testPollVpnConnectionStatusKeepsPendingConnectionWhenSessionIsNotFound() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + when(connection.getState()).thenReturn(Site2SiteVpnConnection.State.Pending); + AtomicBoolean transitioned = new AtomicBoolean(); + + NsxServiceImpl service = new NsxServiceImpl() { + @Override + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + return VPN_SESSION_STATUS_NOT_FOUND; + } + + @Override + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + transitioned.set(true); + } + }; + + service.pollVpnConnectionStatus(connection, vpc); + + assertFalse(transitioned.get()); + } + + @Test + public void testPollVpnConnectionStatusMarksMissingConnectedSessionAsError() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + when(connection.getState()).thenReturn(Site2SiteVpnConnection.State.Connected); + AtomicReference transitionedState = new AtomicReference<>(); + + NsxServiceImpl service = new NsxServiceImpl() { + @Override + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + return VPN_SESSION_STATUS_NOT_FOUND; + } + + @Override + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + transitionedState.set(newState); + } + }; + + service.pollVpnConnectionStatus(connection, vpc); + + assertEquals(Site2SiteVpnConnection.State.Error, transitionedState.get()); + } + + @Test + public void testPollVpnConnectionStatusDoesNotTransitionOnQueryFailure() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + when(connection.getId()).thenReturn(11L); + AtomicBoolean transitioned = new AtomicBoolean(); + + NsxServiceImpl service = new NsxServiceImpl() { + @Override + public String getVpnConnectionStatus(Vpc vpc, String connectionUuid) { + throw new CloudRuntimeException("NSX unavailable"); + } + + @Override + protected void transitionVpnConnectionState(Site2SiteVpnConnectionVO connection, VpcVO vpc, + Site2SiteVpnConnection.State observedState, + Site2SiteVpnConnection.State newState) { + transitioned.set(true); + } + }; + + service.pollVpnConnectionStatus(connection, vpc); + + // A transient management-plane error must not turn a valid connection into Error. + assertTrue(!transitioned.get()); + } + + @Test + public void testTransitionVpnConnectionStateIgnoresStaleStatusObservation() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + Site2SiteVpnConnectionVO lock = mock(Site2SiteVpnConnectionVO.class); + Site2SiteVpnConnectionVO current = mock(Site2SiteVpnConnectionVO.class); + VpcVO vpc = mock(VpcVO.class); + when(connection.getId()).thenReturn(11L); + when(lock.getId()).thenReturn(11L); + when(current.getState()).thenReturn(Site2SiteVpnConnection.State.Disconnected); + when(site2SiteVpnConnectionDao.acquireInLockTable(11L)).thenReturn(lock); + when(site2SiteVpnConnectionDao.findById(11L)).thenReturn(current); + + nsxService.transitionVpnConnectionState(connection, vpc, Site2SiteVpnConnection.State.Connecting, + Site2SiteVpnConnection.State.Connected); + + verify(site2SiteVpnConnectionDao, never()).persist(current); + verify(site2SiteVpnConnectionDao).releaseFromLockTable(11L); + } + + @Test + public void testVpnStatusPollerSkipsUnmarkedGatewayRegardlessOfCurrentOffering() { + Site2SiteVpnConnectionVO connection = mockPollableVpnConnection(); + Site2SiteVpnGatewayVO gateway = mockVpnGatewayForPoller(connection); + VpcVO vpc = mock(VpcVO.class); + when(vpcDao.findById(gateway.getVpcId())).thenReturn(vpc); + NsxServiceImpl service = nsxService; + + service.new VpnStatusPollTask().runInContext(); + + verify(service, never()).pollVpnConnectionStatus(connection, vpc); + } + + @Test + public void testVpnStatusPollerUsesPersistedOwnershipAfterOfferingChanges() { + Site2SiteVpnConnectionVO connection = mockPollableVpnConnection(); + Site2SiteVpnGatewayVO gateway = mockVpnGatewayForPoller(connection); + VpcVO vpc = mock(VpcVO.class); + when(vpcDao.findById(gateway.getVpcId())).thenReturn(vpc); + when(userIpAddressDetailsDao.findDetail(gateway.getAddrId(), NsxElement.NSX_VPN_GATEWAY_IP_DETAIL)) + .thenReturn(mock(UserIpAddressDetailVO.class)); + NsxServiceImpl service = nsxService; + doNothing().when(service).pollVpnConnectionStatus(connection, vpc); + + service.new VpnStatusPollTask().runInContext(); + + verify(service).pollVpnConnectionStatus(connection, vpc); + } + + @Test + public void testVpnStatusPollerQueriesOnlyPollableStates() { + nsxService.new VpnStatusPollTask().runInContext(); + + verify(site2SiteVpnConnectionDao).listByStates( + Site2SiteVpnConnection.State.Pending, + Site2SiteVpnConnection.State.Connecting, + Site2SiteVpnConnection.State.Connected, + Site2SiteVpnConnection.State.Disconnected); + verify(site2SiteVpnConnectionDao, never()).listAll(); + } + + @Test + public void testVpnStatusPollerCanRestartInSameJvm() throws Exception { + ScheduledExecutorService firstExecutor = mock(ScheduledExecutorService.class); + ScheduledExecutorService secondExecutor = mock(ScheduledExecutorService.class); + AtomicInteger executorIndex = new AtomicInteger(); + NsxServiceImpl service = new NsxServiceImpl() { + @Override + protected ScheduledExecutorService createVpnStatusPollExecutor() { + return executorIndex.getAndIncrement() == 0 ? firstExecutor : secondExecutor; + } + }; + service.configure("NsxService", Map.of()); + try { + assertTrue(service.start()); + verify(firstExecutor).scheduleWithFixedDelay(any(Runnable.class), eq(60L), eq(60L), eq(java.util.concurrent.TimeUnit.SECONDS)); + + assertTrue(service.stop()); + verify(firstExecutor).shutdownNow(); + + assertTrue(service.start()); + verify(secondExecutor).scheduleWithFixedDelay(any(Runnable.class), eq(60L), eq(60L), eq(java.util.concurrent.TimeUnit.SECONDS)); + assertEquals(2, executorIndex.get()); + } finally { + service.stop(); + } + verify(secondExecutor).shutdownNow(); + verify(firstExecutor, times(1)).shutdownNow(); + } + + private Site2SiteVpnConnectionVO mockPollableVpnConnection() { + Site2SiteVpnConnectionVO connection = mock(Site2SiteVpnConnectionVO.class); + when(connection.getId()).thenReturn(11L); + when(connection.getVpnGatewayId()).thenReturn(7L); + when(site2SiteVpnConnectionDao.listByStates( + Site2SiteVpnConnection.State.Pending, + Site2SiteVpnConnection.State.Connecting, + Site2SiteVpnConnection.State.Connected, + Site2SiteVpnConnection.State.Disconnected)).thenReturn(List.of(connection)); + return connection; + } + + private Site2SiteVpnGatewayVO mockVpnGatewayForPoller(Site2SiteVpnConnectionVO connection) { + Site2SiteVpnGatewayVO gateway = mock(Site2SiteVpnGatewayVO.class); + when(gateway.getVpcId()).thenReturn(9L); + when(gateway.getAddrId()).thenReturn(30L); + when(site2SiteVpnGatewayDao.findById(connection.getVpnGatewayId())).thenReturn(gateway); + return gateway; + } } diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxControllerUtilsTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxControllerUtilsTest.java index 9139fdef68f7..9168cd04960b 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxControllerUtilsTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxControllerUtilsTest.java @@ -84,6 +84,31 @@ public void testSendCommand() { Assert.assertNotNull(nsxAnswer); } + @Test(expected = InvalidParameterValueException.class) + public void testSendCommandRejectsFailedNsxAnswer() { + NsxCommand cmd = Mockito.mock(NsxCommand.class); + NsxAnswer answer = Mockito.mock(NsxAnswer.class); + Mockito.when(answer.getResult()).thenReturn(false); + Mockito.when(agentMgr.easySend(nsxProviderHostId, cmd)).thenReturn(answer); + + nsxControllerUtils.sendNsxCommand(cmd, zoneId); + } + + @Test + public void testSendCommandForResultPreservesFailedNsxAnswer() { + NsxCommand cmd = Mockito.mock(NsxCommand.class); + NsxAnswer answer = Mockito.mock(NsxAnswer.class); + Mockito.when(answer.getResult()).thenReturn(false); + Mockito.when(answer.isEndpointMayBeInUse()).thenReturn(true); + Mockito.when(agentMgr.easySend(nsxProviderHostId, cmd)).thenReturn(answer); + + NsxAnswer nsxAnswer = nsxControllerUtils.sendNsxCommandForResult(cmd, zoneId); + + Assert.assertSame(answer, nsxAnswer); + Assert.assertFalse(nsxAnswer.getResult()); + Assert.assertTrue(nsxAnswer.isEndpointMayBeInUse()); + } + @Test public void testGetNsxNatRuleIdForVpc() { long vpcId = 5L; diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxHelperTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxHelperTest.java new file mode 100644 index 000000000000..8d32e2499110 --- /dev/null +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxHelperTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils; + +import org.junit.Test; + +import com.cloud.utils.Pair; + +import static org.junit.Assert.assertEquals; + +public class NsxHelperTest { + + @Test + public void testVpnVtiAddressPairFirstSlot() { + Pair vtiAddresses = NsxHelper.getVpnVtiAddressPair(0L); + assertEquals("169.254.64.1", vtiAddresses.first()); + assertEquals("169.254.64.2", vtiAddresses.second()); + } + + @Test + public void testVpnVtiAddressPairIsDerivedFromConnectionId() { + Pair vtiAddresses = NsxHelper.getVpnVtiAddressPair(5L); + assertEquals("169.254.64.21", vtiAddresses.first()); + assertEquals("169.254.64.22", vtiAddresses.second()); + } + + @Test + public void testVpnVtiAddressPairLastSlotStaysWithinSubnet() { + Pair vtiAddresses = NsxHelper.getVpnVtiAddressPair(4095L); + assertEquals("169.254.127.253", vtiAddresses.first()); + assertEquals("169.254.127.254", vtiAddresses.second()); + } + + @Test + public void testVpnVtiAddressPairWrapsAfterSubnetIsExhausted() { + assertEquals(NsxHelper.getVpnVtiAddressPair(1L), NsxHelper.getVpnVtiAddressPair(4097L)); + } + +} diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxVpnCryptoUtilsTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxVpnCryptoUtilsTest.java new file mode 100644 index 000000000000..3eb93ff3d065 --- /dev/null +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/utils/NsxVpnCryptoUtilsTest.java @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils; + +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; + +import com.cloud.exception.InvalidParameterValueException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class NsxVpnCryptoUtilsTest { + + @Test + public void testEncryptionAlgorithmMapping() { + assertEquals(List.of("AES_128"), NsxVpnCryptoUtils.getEncryptionAlgorithms("aes128-sha256;modp2048")); + assertEquals(List.of("AES_256"), NsxVpnCryptoUtils.getEncryptionAlgorithms("aes256-sha1")); + assertEquals(List.of("AES_128", "AES_256"), NsxVpnCryptoUtils.getEncryptionAlgorithms("aes128-sha256,aes256-sha512;modp2048")); + } + + @Test + public void testDigestAlgorithmMapping() { + assertEquals(List.of("SHA1"), NsxVpnCryptoUtils.getDigestAlgorithms("aes128-sha1")); + assertEquals(List.of("SHA2_256"), NsxVpnCryptoUtils.getDigestAlgorithms("aes128-sha256")); + assertEquals(List.of("SHA2_384"), NsxVpnCryptoUtils.getDigestAlgorithms("aes128-sha384")); + assertEquals(List.of("SHA2_512"), NsxVpnCryptoUtils.getDigestAlgorithms("aes128-sha512")); + } + + @Test + public void testDhGroupMapping() { + assertEquals(List.of("GROUP2"), NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp1024")); + assertEquals(List.of("GROUP5"), NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp1536")); + assertEquals(List.of("GROUP14"), NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp2048")); + assertEquals(List.of("GROUP15"), NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp3072")); + assertEquals(List.of("GROUP16"), NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp4096")); + } + + @Test + public void testDhGroupsEmptyWhenPolicyHasNoDhPart() { + assertTrue(NsxVpnCryptoUtils.getDhGroups("aes128-sha256").isEmpty()); + } + + @Test + public void testIkeMultiProposalPolicyWithDhGroupPerEntry() { + String policy = "aes128-sha1;modp2048,aes256-sha256;modp2048"; + assertEquals(List.of("AES_128", "AES_256"), NsxVpnCryptoUtils.getEncryptionAlgorithms(policy)); + assertEquals(List.of("SHA1", "SHA2_256"), NsxVpnCryptoUtils.getDigestAlgorithms(policy)); + assertEquals(List.of("GROUP14"), NsxVpnCryptoUtils.getDhGroups(policy)); + } + + @Test + public void testMultiProposalPolicyCollectsAllDhGroups() { + String policy = "aes128-sha1;modp2048,aes256-sha256;modp3072"; + assertEquals(List.of("GROUP14", "GROUP15"), NsxVpnCryptoUtils.getDhGroups(policy)); + } + + @Test + public void testEspPolicyWithoutDhGroup() { + assertEquals(List.of("AES_128"), NsxVpnCryptoUtils.getEncryptionAlgorithms("aes128-sha256")); + assertEquals(List.of("SHA2_256"), NsxVpnCryptoUtils.getDigestAlgorithms("aes128-sha256")); + assertTrue(NsxVpnCryptoUtils.getDhGroups("aes128-sha256").isEmpty()); + } + + @Test + public void testIkeVersionMapping() { + assertEquals("IKE_FLEX", NsxVpnCryptoUtils.getIkeVersion("ike")); + assertEquals("IKE_FLEX", NsxVpnCryptoUtils.getIkeVersion(null)); + assertEquals("IKE_V1", NsxVpnCryptoUtils.getIkeVersion("ikev1")); + assertEquals("IKE_V2", NsxVpnCryptoUtils.getIkeVersion("ikev2")); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejects3des() { + NsxVpnCryptoUtils.getEncryptionAlgorithms("3des-sha1;modp2048"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsAes192() { + NsxVpnCryptoUtils.getEncryptionAlgorithms("aes192-sha256;modp2048"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsMd5() { + NsxVpnCryptoUtils.getDigestAlgorithms("aes128-md5;modp2048"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsModp6144() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp6144"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsModp8192() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp8192"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsModp1024s160() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp1024s160"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsModp2048s224() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp2048s224"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsModp2048s256() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;modp2048s256"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsCurve25519() { + NsxVpnCryptoUtils.getDhGroups("aes128-sha256;curve25519"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsUnsupportedIkeVersion() { + NsxVpnCryptoUtils.getIkeVersion("ikev3"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsIkeLifetimeBelowNsxMinimum() { + NsxVpnCryptoUtils.validateIkeLifetime(3600L); + } + + @Test + public void testAcceptsIkeLifetimeAtNsxMinimum() { + NsxVpnCryptoUtils.validateIkeLifetime(21600L); + NsxVpnCryptoUtils.validateIkeLifetime(86400L); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsEspLifetimeBelowNsxMinimum() { + NsxVpnCryptoUtils.validateEspLifetime(300L); + } + + @Test + public void testAcceptsDefaultEspLifetime() { + NsxVpnCryptoUtils.validateEspLifetime(3600L); + } + + @Test(expected = InvalidParameterValueException.class) + public void testRejectsPresharedKeyOverNsxMaximumLength() { + NsxVpnCryptoUtils.validatePresharedKey(StringUtils.repeat('k', 129)); + } + + @Test + public void testValidateAcceptsSupportedParameters() { + NsxVpnCryptoUtils.validate("aes256-sha256;modp2048", "aes128-sha1", "ikev2", 86400L, 3600L, "presharedkey"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateRejectsUnsupportedEspPolicy() { + NsxVpnCryptoUtils.validate("aes256-sha256;modp2048", "3des-md5", "ikev2", 86400L, 3600L, "presharedkey"); + } +} diff --git a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java index 804f29c01b1e..3247a43c817c 100644 --- a/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java +++ b/plugins/network-elements/ovs/src/main/java/com/cloud/network/ovs/OvsTunnelManagerImpl.java @@ -396,6 +396,12 @@ boolean isVpcEnabledForDistributedRouter(long vpcId) { return vpc.usesDistributedRouter(); } + boolean isOvsDistributedRouterVpc(long vpcId) { + VpcVO vpc = _vpcDao.findById(vpcId); + return vpc != null && vpc.usesDistributedRouter() + && _vpcMgr.isProviderSupportServiceInVpc(vpcId, Network.Service.Connectivity, Network.Provider.Ovs); + } + @Override public void checkAndPrepareHostForTunnelNetwork(Network nw, Host host) { if (nw.getVpcId() != null && isVpcEnabledForDistributedRouter(nw.getVpcId())) { @@ -684,25 +690,28 @@ private void handleVmStateChange(VMInstanceVO vm) { } for (Long vpcId: vpcIds) { - VpcVO vpc = _vpcDao.findById(vpcId); - // nothing to do if the VPC is not setup for distributed routing - if (vpc == null || !vpc.usesDistributedRouter()) { - return; + if (!isOvsDistributedRouterVpc(vpcId)) { + continue; } - // get the list of hosts on which VPC spans (i.e hosts that need to be aware of VPC topology change update) - List vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId); - String bridgeName=generateBridgeNameForVpc(vpcId); - - OvsVpcPhysicalTopologyConfigCommand topologyConfigCommand = prepareVpcTopologyUpdate(vpcId); - topologyConfigCommand.setSequenceNumber(getNextTopologyUpdateSequenceNumber(vpcId)); - - // send topology change update to VPC spanned hosts - for (Long id: vpcSpannedHostIds) { - if (!sendVpcTopologyChangeUpdate(topologyConfigCommand, id, bridgeName)) { - logger.debug("Failed to send VPC topology change update to host : " + id + ". Moving on " + - "with rest of the host update."); + try { + // get the list of hosts on which VPC spans (i.e hosts that need to be aware of VPC topology change update) + List vpcSpannedHostIds = _ovsNetworkToplogyGuru.getVpcSpannedHosts(vpcId); + String bridgeName=generateBridgeNameForVpc(vpcId); + + OvsVpcPhysicalTopologyConfigCommand topologyConfigCommand = prepareVpcTopologyUpdate(vpcId); + topologyConfigCommand.setSequenceNumber(getNextTopologyUpdateSequenceNumber(vpcId)); + + // send topology change update to VPC spanned hosts + for (Long id: vpcSpannedHostIds) { + if (!sendVpcTopologyChangeUpdate(topologyConfigCommand, id, bridgeName)) { + logger.debug("Failed to send VPC topology change update to host : " + id + ". Moving on " + + "with rest of the host update."); + } } + } catch (RuntimeException e) { + logger.error("Failed to update OVS distributed-router topology for VPC {} after VM {} changed state", + vpcId, vm.getId(), e); } } } @@ -754,20 +763,32 @@ OvsVpcPhysicalTopologyConfigCommand prepareVpcTopologyUpdate(long vpcId) { } for (Network network: vpcNetworks) { + if (network.getBroadcastDomainType() != BroadcastDomainType.Vswitch || network.getBroadcastUri() == null) { + throw new CloudRuntimeException(String.format( + "OVS distributed-router VPC %s contains network %s without a Vswitch broadcast URI", + vpc.getUuid(), network.getUuid())); + } String key = network.getBroadcastUri().getAuthority(); - long gre_key; - if (key.contains(".")) { - String[] parts = key.split("\\."); - gre_key = Long.parseLong(parts[1]); - } else { - try { - gre_key = Long.parseLong(BroadcastDomainType.getValue(key)); - } catch (Exception e) { - return null; - } + String expectedPrefix = vpcId + "."; + if (key == null || !key.startsWith(expectedPrefix) || key.indexOf('.', expectedPrefix.length()) >= 0) { + throw new CloudRuntimeException(String.format( + "OVS distributed-router network %s has invalid broadcast key %s for VPC %s", + network.getUuid(), key, vpc.getUuid())); + } + int greKey; + try { + greKey = Integer.parseInt(key.substring(expectedPrefix.length())); + } catch (NumberFormatException e) { + throw new CloudRuntimeException(String.format( + "OVS distributed-router network %s has non-numeric GRE key %s", + network.getUuid(), key.substring(expectedPrefix.length())), e); } NicVO nic = _nicDao.findByIp4AddressAndNetworkId(network.getGateway(), network.getId()); - OvsVpcPhysicalTopologyConfigCommand.Tier tier = new OvsVpcPhysicalTopologyConfigCommand.Tier(gre_key, + if (nic == null) { + throw new CloudRuntimeException(String.format( + "Unable to find the gateway NIC for OVS distributed-router network %s", network.getUuid())); + } + OvsVpcPhysicalTopologyConfigCommand.Tier tier = new OvsVpcPhysicalTopologyConfigCommand.Tier(greKey, network.getUuid(), network.getGateway(), nic.getMacAddress(), network.getCidr()); tiers.add(tier); } @@ -802,9 +823,9 @@ public class NetworkAclEventsSubscriber implements MessageSubscriber { public void onPublishMessage(String senderAddress, String subject, Object args) { try { NetworkVO network = (NetworkVO) args; - String bridgeName=generateBridgeNameForVpc(network.getVpcId()); - if (network.getVpcId() != null && isVpcEnabledForDistributedRouter(network.getVpcId())) { - long vpcId = network.getVpcId(); + Long vpcId = network.getVpcId(); + if (vpcId != null && isOvsDistributedRouterVpc(vpcId)) { + String bridgeName = generateBridgeNameForVpc(vpcId); OvsVpcRoutingPolicyConfigCommand cmd = prepareVpcRoutingPolicyUpdate(vpcId); cmd.setSequenceNumber(getNextRoutingPolicyUpdateSequenceNumber(vpcId)); diff --git a/plugins/network-elements/ovs/src/test/java/com/cloud/network/ovs/OvsTunnelManagerImplTest.java b/plugins/network-elements/ovs/src/test/java/com/cloud/network/ovs/OvsTunnelManagerImplTest.java new file mode 100644 index 000000000000..a1dbdd403e7d --- /dev/null +++ b/plugins/network-elements/ovs/src/test/java/com/cloud/network/ovs/OvsTunnelManagerImplTest.java @@ -0,0 +1,379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.network.ovs; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.OvsVpcPhysicalTopologyConfigCommand; +import com.cloud.host.dao.HostDao; +import com.cloud.network.Network; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.ovs.dao.VpcDistributedRouterSeqNoDao; +import com.cloud.network.ovs.dao.VpcDistributedRouterSeqNoVO; +import com.cloud.network.vpc.VpcManager; +import com.cloud.network.vpc.VpcVO; +import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.StateMachine2; +import com.cloud.vm.NicVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.VMInstanceDao; + +public class OvsTunnelManagerImplTest { + private static final long VPC_ID = 7L; + private static final long SECOND_VPC_ID = 8L; + + private OvsTunnelManagerImpl manager; + private VpcDao vpcDao; + private VpcManager vpcManager; + private OvsNetworkTopologyGuru topologyGuru; + private NicDao nicDao; + private VpcDistributedRouterSeqNoVO sequenceNumber; + + @Before + public void setUp() { + manager = new OvsTunnelManagerImpl(); + vpcDao = mock(VpcDao.class); + vpcManager = mock(VpcManager.class); + topologyGuru = mock(OvsNetworkTopologyGuru.class); + nicDao = mock(NicDao.class); + manager._vpcDao = vpcDao; + manager._vpcMgr = vpcManager; + manager._ovsNetworkToplogyGuru = topologyGuru; + manager._nicDao = nicDao; + manager._hostDao = mock(HostDao.class); + manager._vmInstanceDao = mock(VMInstanceDao.class); + manager._networkDao = mock(NetworkDao.class); + manager._vpcDrSeqNoDao = mock(VpcDistributedRouterSeqNoDao.class); + manager._agentMgr = mock(AgentManager.class); + } + + @Test + public void testIsOvsDistributedRouterVpcReturnsFalseWhenVpcIsMissing() { + assertFalse(manager.isOvsDistributedRouterVpc(VPC_ID)); + verify(vpcManager, never()).isProviderSupportServiceInVpc(anyLong(), + org.mockito.ArgumentMatchers.any(Network.Service.class), + org.mockito.ArgumentMatchers.any(Network.Provider.class)); + } + + @Test + public void testIsOvsDistributedRouterVpcReturnsFalseWhenVpcIsNotDistributed() { + VpcVO vpc = mock(VpcVO.class); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(false); + + assertFalse(manager.isOvsDistributedRouterVpc(VPC_ID)); + } + + @Test + public void testIsOvsDistributedRouterVpcReturnsFalseForNsxDistributedVpc() { + VpcVO vpc = mock(VpcVO.class); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + + assertFalse(manager.isOvsDistributedRouterVpc(VPC_ID)); + } + + @Test + public void testIsOvsDistributedRouterVpcReturnsTrueForOvsConnectivityDistributedVpc() { + VpcVO vpc = mock(VpcVO.class); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(true); + + assertTrue(manager.isOvsDistributedRouterVpc(VPC_ID)); + } + + @Test + public void testPostStateTransitionEventIgnoresNsxDistributedVpc() { + VpcVO vpc = mock(VpcVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + @SuppressWarnings("unchecked") + StateMachine2.Transition transition = mock(StateMachine2.Transition.class); + when(vm.getId()).thenReturn(11L); + when(topologyGuru.getVpcIdsVmIsPartOf(11L)).thenReturn(List.of(VPC_ID)); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + when(transition.getCurrentState()).thenReturn(VirtualMachine.State.Starting); + when(transition.getEvent()).thenReturn(VirtualMachine.Event.OperationSucceeded); + when(transition.getToState()).thenReturn(VirtualMachine.State.Running); + + assertTrue(manager.postStateTransitionEvent(transition, vm, true, null)); + + verify(topologyGuru, never()).getVpcSpannedHosts(anyLong()); + verify(vpcManager, never()).getVpcNetworks(anyLong()); + } + + @Test + public void testPostStateTransitionEventContinuesAfterNonOvsVpc() { + VpcVO firstVpc = mock(VpcVO.class); + VpcVO secondVpc = mock(VpcVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + @SuppressWarnings("unchecked") + StateMachine2.Transition transition = mock(StateMachine2.Transition.class); + when(vm.getId()).thenReturn(11L); + when(topologyGuru.getVpcIdsVmIsPartOf(11L)).thenReturn(List.of(VPC_ID, SECOND_VPC_ID)); + when(vpcDao.findById(VPC_ID)).thenReturn(firstVpc); + when(vpcDao.findById(SECOND_VPC_ID)).thenReturn(secondVpc); + when(firstVpc.usesDistributedRouter()).thenReturn(true); + when(secondVpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + when(vpcManager.isProviderSupportServiceInVpc(SECOND_VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + when(transition.getCurrentState()).thenReturn(VirtualMachine.State.Starting); + when(transition.getEvent()).thenReturn(VirtualMachine.Event.OperationSucceeded); + when(transition.getToState()).thenReturn(VirtualMachine.State.Running); + + assertTrue(manager.postStateTransitionEvent(transition, vm, true, null)); + + verify(vpcDao).findById(SECOND_VPC_ID); + } + + @Test + public void testPostStateTransitionEventContainsMalformedOvsTopologyAndContinues() { + VpcVO firstVpc = mock(VpcVO.class); + VpcVO secondVpc = mock(VpcVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + @SuppressWarnings("unchecked") + StateMachine2.Transition transition = mock(StateMachine2.Transition.class); + when(vm.getId()).thenReturn(11L); + when(topologyGuru.getVpcIdsVmIsPartOf(11L)).thenReturn(List.of(VPC_ID, SECOND_VPC_ID)); + when(vpcDao.findById(VPC_ID)).thenReturn(firstVpc); + when(vpcDao.findById(SECOND_VPC_ID)).thenReturn(secondVpc); + when(firstVpc.usesDistributedRouter()).thenReturn(true); + when(secondVpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(SECOND_VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + when(transition.getCurrentState()).thenReturn(VirtualMachine.State.Starting); + when(transition.getEvent()).thenReturn(VirtualMachine.Event.OperationSucceeded); + when(transition.getToState()).thenReturn(VirtualMachine.State.Running); + Network malformedNetwork = mock(Network.class); + when(topologyGuru.getVpcSpannedHosts(VPC_ID)).thenReturn(Collections.emptyList()); + when(topologyGuru.getAllActiveVmsInVpc(VPC_ID)).thenReturn(Collections.emptyList()); + doReturn(List.of(malformedNetwork)).when(vpcManager).getVpcNetworks(VPC_ID); + when(firstVpc.getUuid()).thenReturn("vpc-uuid"); + when(firstVpc.getCidr()).thenReturn("10.0.0.0/16"); + when(malformedNetwork.getUuid()).thenReturn("network-uuid"); + when(malformedNetwork.getBroadcastDomainType()).thenReturn(BroadcastDomainType.NSX); + + assertTrue(manager.postStateTransitionEvent(transition, vm, true, null)); + + verify(vpcDao).findById(SECOND_VPC_ID); + } + + @Test + public void testPostStateTransitionEventBuildsTopologyForOvsVpc() { + VpcVO vpc = mock(VpcVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + @SuppressWarnings("unchecked") + StateMachine2.Transition transition = mock(StateMachine2.Transition.class); + when(vm.getId()).thenReturn(11L); + when(topologyGuru.getVpcIdsVmIsPartOf(11L)).thenReturn(List.of(VPC_ID)); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpc.getUuid()).thenReturn("vpc-uuid"); + when(vpc.getCidr()).thenReturn("10.0.0.0/16"); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(true); + when(transition.getCurrentState()).thenReturn(VirtualMachine.State.Starting); + when(transition.getEvent()).thenReturn(VirtualMachine.Event.OperationSucceeded); + when(transition.getToState()).thenReturn(VirtualMachine.State.Running); + when(topologyGuru.getVpcSpannedHosts(VPC_ID)).thenReturn(Collections.emptyList()); + when(topologyGuru.getAllActiveVmsInVpc(VPC_ID)).thenReturn(Collections.emptyList()); + doReturn(Collections.emptyList()).when(vpcManager).getVpcNetworks(VPC_ID); + prepareSequenceNumber(VPC_ID); + + assertTrue(manager.postStateTransitionEvent(transition, vm, true, null)); + + verify(vpcManager).getVpcNetworks(VPC_ID); + verify(manager._vpcDrSeqNoDao).update(1L, sequenceNumber); + } + + @Test + public void testNetworkAclSubscriberIgnoresNsxDistributedVpc() { + VpcVO vpc = mock(VpcVO.class); + NetworkVO network = mock(NetworkVO.class); + when(network.getVpcId()).thenReturn(VPC_ID); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(false); + + manager.new NetworkAclEventsSubscriber().onPublishMessage("sender", "Network_ACL_Replaced", network); + + verify(topologyGuru, never()).getVpcSpannedHosts(anyLong()); + verify(vpcManager, never()).getVpcNetworks(anyLong()); + } + + @Test + public void testNetworkAclSubscriberBuildsPolicyForOvsVpc() { + VpcVO vpc = mock(VpcVO.class); + NetworkVO network = mock(NetworkVO.class); + when(network.getVpcId()).thenReturn(VPC_ID); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.usesDistributedRouter()).thenReturn(true); + when(vpc.getUuid()).thenReturn("vpc-uuid"); + when(vpc.getCidr()).thenReturn("10.0.0.0/16"); + when(vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Connectivity, Network.Provider.Ovs)) + .thenReturn(true); + doReturn(List.of(network)).when(vpcManager).getVpcNetworks(VPC_ID); + when(network.getNetworkACLId()).thenReturn(null); + when(topologyGuru.getVpcSpannedHosts(VPC_ID)).thenReturn(Collections.emptyList()); + prepareSequenceNumber(VPC_ID); + + manager.new NetworkAclEventsSubscriber().onPublishMessage("sender", "Network_ACL_Replaced", network); + + verify(vpcManager).getVpcNetworks(VPC_ID); + verify(manager._vpcDrSeqNoDao).update(1L, sequenceNumber); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsNonVswitchTier() { + VpcVO vpc = mock(VpcVO.class); + Network network = mock(Network.class); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.getUuid()).thenReturn("vpc-uuid"); + doReturn(List.of(network)).when(vpcManager).getVpcNetworks(VPC_ID); + when(topologyGuru.getVpcSpannedHosts(VPC_ID)).thenReturn(Collections.emptyList()); + when(topologyGuru.getAllActiveVmsInVpc(VPC_ID)).thenReturn(Collections.emptyList()); + when(network.getUuid()).thenReturn("network-uuid"); + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.NSX); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsBroadcastKeyForAnotherVpc() { + Network network = prepareVswitchNetwork("8.123"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + + verify(nicDao, never()).findByIp4AddressAndNetworkId("10.0.1.1", 13L); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsNonNumericGreKey() { + prepareVswitchNetwork("7.invalid"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsRepeatedDelimiterInBroadcastKey() { + prepareVswitchNetwork("7..123"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsLeadingDelimiterInBroadcastKey() { + prepareVswitchNetwork(".7.123"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsTrailingDelimiterInBroadcastKey() { + prepareVswitchNetwork("7.123."); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsGreKeyOutsideIntegerRange() { + prepareVswitchNetwork("7.2147483648"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateRejectsMissingGatewayNic() { + prepareVswitchNetwork("7.123"); + + assertThrows(CloudRuntimeException.class, () -> manager.prepareVpcTopologyUpdate(VPC_ID)); + } + + @Test + public void testPrepareVpcTopologyUpdateBuildsValidOvsTopology() { + prepareVswitchNetwork("7.123"); + NicVO gatewayNic = mock(NicVO.class); + when(nicDao.findByIp4AddressAndNetworkId("10.0.1.1", 13L)).thenReturn(gatewayNic); + when(gatewayNic.getMacAddress()).thenReturn("02:00:00:00:00:01"); + + OvsVpcPhysicalTopologyConfigCommand command = manager.prepareVpcTopologyUpdate(VPC_ID); + + String topology = command.getVpcConfigInJson(); + assertTrue(topology.contains("\"grekey\":123")); + assertTrue(topology.contains("\"networkuuid\":\"network-uuid\"")); + assertTrue(topology.contains("\"gatewaymac\":\"02:00:00:00:00:01\"")); + } + + private Network prepareVswitchNetwork(String broadcastKey) { + VpcVO vpc = mock(VpcVO.class); + Network network = mock(Network.class); + when(vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(vpc.getUuid()).thenReturn("vpc-uuid"); + when(vpc.getCidr()).thenReturn("10.0.0.0/16"); + doReturn(List.of(network)).when(vpcManager).getVpcNetworks(VPC_ID); + when(topologyGuru.getVpcSpannedHosts(VPC_ID)).thenReturn(Collections.emptyList()); + when(topologyGuru.getAllActiveVmsInVpc(VPC_ID)).thenReturn(Collections.emptyList()); + when(network.getId()).thenReturn(13L); + when(network.getUuid()).thenReturn("network-uuid"); + when(network.getGateway()).thenReturn("10.0.1.1"); + when(network.getCidr()).thenReturn("10.0.1.0/24"); + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Vswitch); + when(network.getBroadcastUri()).thenReturn(BroadcastDomainType.Vswitch.toUri(broadcastKey)); + return network; + } + + private void prepareSequenceNumber(long vpcId) { + sequenceNumber = mock(VpcDistributedRouterSeqNoVO.class); + when(sequenceNumber.getId()).thenReturn(1L); + when(sequenceNumber.getTopologyUpdateSequenceNo()).thenReturn(1L); + when(sequenceNumber.getPolicyUpdateSequenceNo()).thenReturn(1L); + when(manager._vpcDrSeqNoDao.findByVpcId(vpcId)).thenReturn(sequenceNumber); + when(manager._vpcDrSeqNoDao.lockRow(1L, true)).thenReturn(sequenceNumber); + } +} diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index 957366ad22d7..65b4c74f8d9c 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -85,7 +85,7 @@ public class ApiServlet extends HttpServlet { private static final String REPLACEMENT = "_"; private static final String LOGGER_REPLACEMENTS = "[\n\r\t]"; public static final Pattern GET_REQUEST_COMMANDS = Pattern.compile("^(get|list|query|find)(\\w+)+$"); - private static final HashSet POST_REQUESTS_TO_DISABLE_LOGGING = new HashSet<>(Set.of( + private static final HashSet REQUESTS_TO_DISABLE_PARAMETER_LOGGING = new HashSet<>(Set.of( "login", "oauthlogin", "createaccount", @@ -97,6 +97,7 @@ public class ApiServlet extends HttpServlet { "updaterolepermission", "updateprojectrolepermission", "createstoragepool", + "addobjectstoragepool", "addhost", "updatehostpassword", "addcluster", @@ -234,17 +235,15 @@ void processRequestInContext(final HttpServletRequest req, final HttpServletResp // logging the request start and end in management log for easy debugging String reqStr = ""; - String cleanQueryString = StringUtils.cleanString(req.getQueryString()); + String cleanQueryString = getCleanQueryString(command, req.getQueryString(), reqParams); if (LOGGER.isDebugEnabled()) { reqStr = auditTrailSb.toString() + " " + cleanQueryString; if (req.getMethod().equalsIgnoreCase("POST") && org.apache.commons.lang3.StringUtils.isNotBlank(command)) { - if (!POST_REQUESTS_TO_DISABLE_LOGGING.contains(command.toLowerCase()) && !reqParams.containsKey(ApiConstants.USER_DATA)) { + if (shouldLogRequestParameters(command, reqParams)) { String cleanParamsString = getCleanParamsString(reqParams); if (org.apache.commons.lang3.StringUtils.isNotBlank(cleanParamsString)) { reqStr += "\n" + cleanParamsString; } - } else { - reqStr += " " + command; } } LOGGER.debug("===START=== " + reqStr); @@ -787,4 +786,17 @@ private String getCleanParamsString(Map reqParams) { return cleanParamsString.toString(); } + + protected boolean shouldLogRequestParameters(String command, Map reqParams) { + return (org.apache.commons.lang3.StringUtils.isBlank(command) + || !REQUESTS_TO_DISABLE_PARAMETER_LOGGING.contains(command.toLowerCase(java.util.Locale.ROOT))) + && !reqParams.containsKey(ApiConstants.USER_DATA); + } + + protected String getCleanQueryString(String command, String queryString, Map reqParams) { + if (!shouldLogRequestParameters(command, reqParams)) { + return org.apache.commons.lang3.StringUtils.isBlank(command) ? "" : "command=" + saveLogString(command); + } + return StringUtils.cleanString(queryString); + } } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 837254ed8b36..e50872196fa4 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -251,6 +251,8 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao; import com.cloud.network.dao.PhysicalNetworkTrafficTypeVO; @@ -258,6 +260,8 @@ import com.cloud.network.dao.UserIpv6AddressDao; import com.cloud.network.element.NetrisProviderVO; import com.cloud.network.element.NsxProviderVO; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.nsx.NsxService; import com.cloud.network.netris.NetrisService; import com.cloud.network.rules.LoadBalancerContainer.Scheme; import com.cloud.network.vpc.VpcManager; @@ -392,6 +396,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati @Inject VlanDetailsDao vlanDetailsDao; @Inject + NsxVrfGatewayDao nsxVrfGatewayDao; + @Inject + NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Inject IPAddressDao _publicIpAddressDao; @Inject DataCenterIpAddressDao _privateIpAddressDao; @@ -572,6 +580,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati private static final String DefaultVlanForPodIpRange = Vlan.UNTAGGED; private static final Set VPC_ONLY_PROVIDERS = Sets.newHashSet(Provider.VPCVirtualRouter, Provider.JuniperContrailVpcRouter, Provider.InternalLbVm); + private static final Set NSX_SEGMENT_PROFILE_DETAILS = Set.of(Detail.NsxIpDiscoveryProfileId, + Detail.NsxMacDiscoveryProfileId, Detail.NsxSegmentSecurityProfileId); + private static final int MAX_NSX_PROFILE_ID_LENGTH = 255; private static final List SUPPORTED_ROUTING_MODE_STRS = Arrays.asList(Static.toString().toLowerCase(), Dynamic.toString().toLowerCase()); private static final long GiB_TO_BYTES = 1024 * 1024 * 1024; @@ -6155,6 +6166,23 @@ public Vlan updateVlanAndPublicIpRange(final long id, String startIp, throw new InvalidParameterValueException("Please specify a valid IP range id."); } + return withNsxVrfZoneLock(vlanRange, lockedVlan -> updateVlanAndPublicIpRangeLocked(id, lockedVlan, + startIp, endIp, gateway, netmask, startIpv6, endIpv6, ip6Gateway, ip6Cidr, forSystemVms)); + } + + private Vlan updateVlanAndPublicIpRangeLocked(final long id, VlanVO vlanRange, String startIp, + String endIp, String gateway, String netmask, String startIpv6, String endIpv6, + String ip6Gateway, String ip6Cidr, Boolean forSystemVms) throws ResourceAllocationException { + NsxVrfGatewayVO vrfGateway = nsxVrfGatewayDao.findByPublicVlan(id); + if (vrfGateway != null && ((!Objects.isNull(gateway) && !Objects.equals(gateway, vlanRange.getVlanGateway())) + || (!Objects.isNull(netmask) && !Objects.equals(netmask, vlanRange.getVlanNetmask())) + || (!Objects.isNull(ip6Gateway) && !Objects.equals(ip6Gateway, vlanRange.getIp6Gateway())) + || (!Objects.isNull(ip6Cidr) && !Objects.equals(ip6Cidr, vlanRange.getIp6Cidr())))) { + throw new InvalidParameterValueException(String.format( + "Cannot change subnet metadata for public IP range %s while it is registered to NSX VRF gateway %s", + id, vrfGateway.getNsxTier0Name())); + } + final boolean ipv4 = vlanRange.getVlanGateway() != null; final boolean ipv6 = vlanRange.getIp6Gateway() != null; if (!ipv4) { @@ -6175,6 +6203,11 @@ public Vlan updateVlanAndPublicIpRange(final long id, String startIp, Long domainId = domainMap != null ? domainMap.getDomainId() : null; final Boolean isRangeForSystemVM = checkIfVlanRangeIsForSystemVM(id); + if (vrfGateway != null && forSystemVms != null && !Objects.equals(isRangeForSystemVM, forSystemVms)) { + throw new InvalidParameterValueException(String.format( + "Cannot change forSystemVms for public IP range %s while it is registered to NSX VRF gateway %s", + id, vrfGateway.getNsxTier0Name())); + } if (forSystemVms != null && isRangeForSystemVM != forSystemVms) { if (VlanType.DirectAttached.equals(vlanRange.getVlanType())) { throw new InvalidParameterValueException("forSystemVms is not available for this IP range with vlan type: " + VlanType.DirectAttached); @@ -6431,6 +6464,19 @@ public VlanVO deleteVlanAndPublicIpRange(final long userId, final long vlanDbId, throw new InvalidParameterValueException("Please specify a valid IP range id."); } + return withNsxVrfZoneLock(vlanRange, + lockedVlan -> deleteVlanAndPublicIpRangeLocked(userId, vlanDbId, caller, lockedVlan)); + } + + private VlanVO deleteVlanAndPublicIpRangeLocked(final long userId, final long vlanDbId, + final Account caller, VlanVO vlanRange) { + NsxVrfGatewayVO vrfGateway = nsxVrfGatewayDao.findByPublicVlan(vlanDbId); + if (vrfGateway != null) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s is registered to NSX VRF gateway %s; delete the gateway registration first", + vlanDbId, vrfGateway.getNsxTier0Name())); + } + boolean isAccountSpecific = false; final List acctVln = _accountVlanMapDao.listAccountVlanMapsByVlan(vlanRange.getId()); // Check for account wide pool. It will have an entry for @@ -6530,6 +6576,33 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { return vlanRange; } + @FunctionalInterface + private interface VlanOperation { + T apply(VlanVO vlan) throws E; + } + + private T withNsxVrfZoneLock(VlanVO vlan, VlanOperation operation) throws E { + GlobalLock lock = GlobalLock.getInternLock(NsxService.getVrfZoneLockName(vlan.getDataCenterId())); + boolean locked = false; + try { + if (!lock.lock(30)) { + throw new CloudRuntimeException(String.format( + "Timed out waiting to modify public IP range %s", vlan.getId())); + } + locked = true; + VlanVO currentVlan = _vlanDao.findById(vlan.getId()); + if (currentVlan == null) { + throw new InvalidParameterValueException("Please specify a valid IP range id."); + } + return operation.apply(currentVlan); + } finally { + if (locked) { + lock.unlock(); + } + lock.releaseRef(); + } + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_VLAN_IP_RANGE_DEDICATE, eventDescription = "dedicating vlan ip range", async = false) @@ -6671,6 +6744,37 @@ public boolean releasePublicIpRange(final long vlanDbId, final User user, final return true; } + GlobalLock vrfZoneLock = GlobalLock.getInternLock(NsxService.getVrfZoneLockName(vlan.getDataCenterId())); + boolean vrfZoneLocked = false; + try { + if (!vrfZoneLock.lock(30)) { + throw new CloudRuntimeException("Timed out waiting to release the public IP range"); + } + vrfZoneLocked = true; + vlan = _vlanDao.findById(vlanDbId); + if (vlan == null) { + logger.warn("Skipping release of public IP range {} because it was removed while waiting for the zone lock", + vlanDbId); + return true; + } + NsxVrfGatewayVO vrfGateway = nsxVrfGatewayDao.findByPublicVlan(vlanDbId); + if (vrfGateway != null && !vrfGateway.isUnclaimed()) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s is registered to assigned NSX VRF gateway %s; release the gateway first", + vlanDbId, vrfGateway.getNsxTier0Name())); + } + return releasePublicIpRangeLocked(vlanDbId, user, caller, vlan); + } finally { + if (vrfZoneLocked) { + vrfZoneLock.unlock(); + } + vrfZoneLock.releaseRef(); + } + } + + private boolean releasePublicIpRangeLocked(final long vlanDbId, final User user, final Account caller, + VlanVO vlan) { + // Verify range is dedicated boolean isAccountSpecific = false; final List acctVln = _accountVlanMapDao.listAccountVlanMapsByVlan(vlanDbId); @@ -6746,6 +6850,41 @@ public boolean releasePublicIpRange(final long vlanDbId, final User user, final return success; } + private boolean releaseOwnerPublicIpRange(final long vlanDbId, final User user, final Account caller, + final Long accountId, final Long domainId) { + VlanVO vlan = _vlanDao.findById(vlanDbId); + if (vlan == null) { + return true; + } + return withNsxVrfZoneLock(vlan, lockedVlan -> { + NsxVrfGatewayVO gateway = nsxVrfGatewayDao.findByPublicVlan(vlanDbId); + if (gateway != null && !gateway.isUnclaimed()) { + boolean ownerMatches = accountId != null + ? Objects.equals(gateway.getAccountId(), accountId) + : Objects.equals(gateway.getDomainId(), domainId); + if (!ownerMatches) { + throw new InvalidParameterValueException(String.format( + "Public IP range %s is assigned to a different NSX VRF gateway owner", vlanDbId)); + } + long placements = nsxVrfGatewayPlacementDao.countByGatewayId(gateway.getId()); + if (placements > 0) { + throw new InvalidParameterValueException(String.format( + "NSX VRF gateway %s still has %d Tier-1 placement(s)", + gateway.getNsxTier0Name(), placements)); + } + gateway.setScope(null); + gateway.setAccountId(null); + gateway.setDomainId(null); + if (!nsxVrfGatewayDao.update(gateway.getId(), gateway)) { + throw new CloudRuntimeException(String.format( + "Failed to release NSX VRF gateway %s during owner cleanup", + gateway.getNsxTier0Name())); + } + } + return releasePublicIpRangeLocked(vlanDbId, user, caller, lockedVlan); + }); + } + @DB protected boolean savePublicIPRange(final String startIP, final String endIP, final long zoneId, final long vlanDbId, final long sourceNetworkid, final long physicalNetworkId, final boolean forSystemVms) { final long startIPLong = NetUtils.ip2Long(startIP); @@ -7987,8 +8126,27 @@ boolean isSharedSourceNat(Map> serviceProviderMap, Map details, final Map> serviceProviderMap) { + Set connectivityProviders = serviceProviderMap == null ? null : serviceProviderMap.get(Service.Connectivity); + boolean nsxConnectivityConfigured = connectivityProviders != null && connectivityProviders.contains(Provider.Nsx); for (final Detail detail : details.keySet()) { + if (NSX_SEGMENT_PROFILE_DETAILS.contains(detail)) { + String profileId = details.get(detail); + if (!nsxConnectivityConfigured) { + throw new InvalidParameterValueException(String.format("Detail %s is supported only by network offerings with NSX connectivity", detail)); + } + if (StringUtils.isBlank(profileId)) { + throw new InvalidParameterValueException(String.format("A non-empty NSX profile ID is required for detail %s", detail)); + } + if (!profileId.equals(profileId.trim())) { + throw new InvalidParameterValueException(String.format("NSX profile ID for detail %s must not contain surrounding whitespace", detail)); + } + if (profileId.length() > MAX_NSX_PROFILE_ID_LENGTH) { + throw new InvalidParameterValueException(String.format("NSX profile ID for detail %s cannot exceed %d characters", + detail, MAX_NSX_PROFILE_ID_LENGTH)); + } + } + Provider lbProvider = null; if (detail == NetworkOffering.Detail.InternalLbProvider || detail == NetworkOffering.Detail.PublicLbProvider) { // 1) Vaidate the detail values - have to match the lb provider @@ -9120,7 +9278,8 @@ public boolean releaseDomainSpecificVirtualRanges(final Domain domain) { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { for (DomainVlanMapVO map : maps) { - if (!releasePublicIpRange(map.getVlanDbId(), _accountMgr.getSystemUser(), _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM))) { + if (!releaseOwnerPublicIpRange(map.getVlanDbId(), _accountMgr.getSystemUser(), + _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM), null, domain.getId())) { throw new CloudRuntimeException(String.format("Failed to release domain specific virtual ip ranges for domain %s", domain)); } } @@ -9146,7 +9305,8 @@ public boolean releaseAccountSpecificVirtualRanges(final Account account) { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { for (final AccountVlanMapVO map : maps) { - if (!releasePublicIpRange(map.getVlanDbId(), _accountMgr.getSystemUser(), _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM))) { + if (!releaseOwnerPublicIpRange(map.getVlanDbId(), _accountMgr.getSystemUser(), + _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM), account.getId(), null)) { throw new CloudRuntimeException(String.format("Failed to release account specific virtual ip ranges for account %s", account)); } } diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 3cd42c50c28d..2c8093160f6d 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -130,6 +130,7 @@ import com.cloud.network.dao.NetworkDetailVO; import com.cloud.network.dao.NetworkDomainDao; import com.cloud.network.dao.NetworkServiceMapDao; +import com.cloud.network.dao.NsxVrfGatewayDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao; @@ -137,9 +138,11 @@ import com.cloud.network.element.IpDeployer; import com.cloud.network.element.IpDeployingRequester; import com.cloud.network.element.NetworkElement; +import com.cloud.network.element.NsxVrfGatewayVO; import com.cloud.network.element.StaticNatServiceProvider; import com.cloud.network.guru.NetworkGuru; import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.nsx.NsxService; import com.cloud.network.rules.FirewallManager; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; @@ -171,6 +174,7 @@ import com.cloud.utils.Journal; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; +import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; @@ -333,6 +337,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Inject NetrisProviderDao netrisProviderDao; @Inject + NsxVrfGatewayDao nsxVrfGatewayDao; + @Inject VlanDetailsDao vlanDetailsDao; @Inject @@ -538,6 +544,7 @@ public boolean configure(String name, Map params) { AssignIpAddressSearch.and("dc", AssignIpAddressSearch.entity().getDataCenterId(), Op.EQ); AssignIpAddressSearch.and("allocated", AssignIpAddressSearch.entity().getAllocatedTime(), Op.NULL); AssignIpAddressSearch.and("vlanId", AssignIpAddressSearch.entity().getVlanId(), Op.IN); + AssignIpAddressSearch.and("excludedVlanId", AssignIpAddressSearch.entity().getVlanId(), Op.NIN); AssignIpAddressSearch.and("forSystemVms", AssignIpAddressSearch.entity().isForSystemVms(), Op.EQ); AssignIpAddressSearch.and("id", AssignIpAddressSearch.entity().getId(), Op.NIN); AssignIpAddressSearch.and("requestedAddress", AssignIpAddressSearch.entity().getAddress(), Op.EQ); @@ -554,6 +561,7 @@ public boolean configure(String name, Map params) { AssignIpAddressFromPodVlanSearch.and("dc", AssignIpAddressFromPodVlanSearch.entity().getDataCenterId(), Op.EQ); AssignIpAddressFromPodVlanSearch.and("allocated", AssignIpAddressFromPodVlanSearch.entity().getAllocatedTime(), Op.NULL); AssignIpAddressFromPodVlanSearch.and("vlanId", AssignIpAddressFromPodVlanSearch.entity().getVlanId(), Op.IN); + AssignIpAddressFromPodVlanSearch.and("excludedVlanId", AssignIpAddressFromPodVlanSearch.entity().getVlanId(), Op.NIN); SearchBuilder podVlanSearch = _vlanDao.createSearchBuilder(); podVlanSearch.and("type", podVlanSearch.entity().getVlanType(), Op.EQ); @@ -950,7 +958,19 @@ public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final String requestedGateway, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms) throws InsufficientAddressCapacityException { - List addrs = listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, allocate, requestedIp, requestedGateway, isSystem, vpcId, displayIp, forSystemVms, true); + return fetchNewPublicIp(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, + allocate, requestedIp, requestedGateway, isSystem, vpcId, displayIp, forSystemVms, true, false); + } + + private PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List vlanDbIds, + final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, + final boolean assign, final boolean allocate, final String requestedIp, final String requestedGateway, + final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms, + final boolean allowSystemPoolFallback, final boolean allowRegisteredNsxVrfVlan) + throws InsufficientAddressCapacityException { + List addrs = listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, + sourceNat, assign, allocate, requestedIp, requestedGateway, isSystem, vpcId, displayIp, + forSystemVms, true, allowSystemPoolFallback, allowRegisteredNsxVrfVlan); IPAddressVO addr = addrs.get(0); if (assign) { addr = assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, @@ -967,12 +987,34 @@ public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List listAvailablePublicIps(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final String requestedGateway, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms, final boolean lockOneRow) throws InsufficientAddressCapacityException { + return listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, + allocate, requestedIp, requestedGateway, isSystem, vpcId, displayIp, forSystemVms, lockOneRow, + true, false); + } + + private List listAvailablePublicIps(final long dcId, final Long podId, + final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, + final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, + final String requestedGateway, final boolean isSystem, final Long vpcId, final Boolean displayIp, + final boolean forSystemVms, final boolean lockOneRow, final boolean allowSystemPoolFallback, + final boolean allowRegisteredNsxVrfVlan) throws InsufficientAddressCapacityException { StringBuilder errorMessage = new StringBuilder("Unable to get ip address in "); boolean fetchFromDedicatedRange = false; List dedicatedVlanDbIds = new ArrayList(); List nonDedicatedVlanDbIds = new ArrayList(); DataCenter zone = _entityMgr.findById(DataCenter.class, dcId); + List registeredNsxVrfVlanIds = nsxVrfGatewayDao.listByZone(dcId).stream() + .map(NsxVrfGatewayVO::getPublicVlanDbId) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + List eligibleVlanDbIds = vlanDbIds; + if (!allowRegisteredNsxVrfVlan && vlanDbIds != null && !registeredNsxVrfVlanIds.isEmpty()) { + eligibleVlanDbIds = vlanDbIds.stream() + .filter(id -> !registeredNsxVrfVlanIds.contains(id)) + .collect(Collectors.toList()); + } SearchCriteria sc = null; if (podId != null) { @@ -998,6 +1040,9 @@ public List listAvailablePublicIps(final long dcId, final Long podI } sc.setParameters("dc", dcId); + if (!allowRegisteredNsxVrfVlan && !registeredNsxVrfVlanIds.isEmpty()) { + sc.setParameters("excludedVlanId", registeredNsxVrfVlanIds.toArray()); + } // for direct network take ip addresses only from the vlans belonging to the network if (vlanUse == VlanType.DirectAttached) { @@ -1049,18 +1094,27 @@ public List listAvailablePublicIps(final long dcId, final Long podI if (network == null || !(network.getGuestType() == GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced)) { List accountVlanMaps = _accountVlanMapDao.listAccountVlanMapsByAccount(owner.getId()); for (AccountVlanMapVO map : accountVlanMaps) { - if (vlanDbIds == null || vlanDbIds.contains(map.getVlanDbId())) + if (eligibleVlanDbIds == null || eligibleVlanDbIds.contains(map.getVlanDbId())) dedicatedVlanDbIds.add(map.getVlanDbId()); } } List domainMaps = _domainVlanMapDao.listDomainVlanMapsByDomain(owner.getDomainId()); for (DomainVlanMapVO map : domainMaps) { - if (vlanDbIds == null || vlanDbIds.contains(map.getVlanDbId())) + if (eligibleVlanDbIds == null || eligibleVlanDbIds.contains(map.getVlanDbId())) dedicatedVlanDbIds.add(map.getVlanDbId()); } + if (eligibleVlanDbIds != null) { + for (Long vlanDbId : eligibleVlanDbIds) { + boolean dedicatedToAncestorDomain = _domainVlanMapDao.listDomainVlanMapsByVlan(vlanDbId).stream() + .anyMatch(map -> _domainDao.isChildDomain(map.getDomainId(), owner.getDomainId())); + if (dedicatedToAncestorDomain && !dedicatedVlanDbIds.contains(vlanDbId)) { + dedicatedVlanDbIds.add(vlanDbId); + } + } + } List nonDedicatedVlans = _vlanDao.listZoneWideNonDedicatedVlans(dcId); for (VlanVO nonDedicatedVlan : nonDedicatedVlans) { - if (vlanDbIds == null || vlanDbIds.contains(nonDedicatedVlan.getId())) + if (eligibleVlanDbIds == null || eligibleVlanDbIds.contains(nonDedicatedVlan.getId())) nonDedicatedVlanDbIds.add(nonDedicatedVlan.getId()); } if (vlanUse == VlanType.VirtualNetwork) { @@ -1090,7 +1144,8 @@ public List listAvailablePublicIps(final long dcId, final Long podI } // If all the dedicated IPs of the owner are in use fetch an IP from the system pool - if ((!lockOneRow || (lockOneRow && addresses.isEmpty())) && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) { + if (allowSystemPoolFallback && (!lockOneRow || (lockOneRow && addresses.isEmpty())) + && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) { // Verify if account is allowed to acquire IPs from the system boolean useSystemIps = UseSystemPublicIps.valueIn(owner.getId()); if (useSystemIps && !nonDedicatedVlanDbIds.isEmpty()) { @@ -1121,6 +1176,7 @@ public List listAvailablePublicIps(final long dcId, final Long podI if (lockOneRow) { IPAddressVO allocatableIp = addresses.get(0); + validateSelectedPublicIpRangeRegistration(allocatableIp, allowRegisteredNsxVrfVlan, dcId); boolean isPublicIpAllocatable = canPublicIpAddressBeAllocated(allocatableIp, owner); @@ -1142,6 +1198,16 @@ public List listAvailablePublicIps(final long dcId, final Long podI return addresses; } + void validateSelectedPublicIpRangeRegistration(IPAddressVO address, boolean allowRegisteredNsxVrfVlan, + long zoneId) throws InsufficientAddressCapacityException { + if (!allowRegisteredNsxVrfVlan + && nsxVrfGatewayDao.lockByPublicVlan(address.getVlanId()) != null) { + throw new InsufficientAddressCapacityException( + "The selected public IP range was registered to an NSX VRF gateway during allocation", + DataCenter.class, zoneId); + } + } + @DB @Override public void markPublicIpAsAllocated(final IPAddressVO addr) { @@ -1213,7 +1279,19 @@ public PublicIp assignSourceNatIpAddressToGuestNetwork(Account owner, Network gu if (sourceNatIp != null) { ipToReturn = PublicIp.createFromAddrAndVlan(sourceNatIp, _vlanDao.findById(sourceNatIp.getVlanId())); } else { - ipToReturn = assignDedicateIpAddress(owner, guestNetwork.getId(), null, dcId, true); + Long publicVlanId = null; + if (guestNetwork.getBroadcastDomainType() == BroadcastDomainType.NSX) { + NsxService nsxService = ComponentContext.getDelegateComponentOfType(NsxService.class); + if (nsxService == null) { + throw new CloudRuntimeException("The NSX service is unavailable"); + } + publicVlanId = nsxService.reserveTier1PlacementAndGetPublicVlanId(dcId, owner.getId(), owner.getDomainId(), + null, guestNetwork.getId()); + } + ipToReturn = publicVlanId == null + ? assignDedicateIpAddress(owner, guestNetwork.getId(), null, dcId, true) + : assignDedicateIpAddressFromNsxVrfPublicRange(owner, guestNetwork.getId(), null, dcId, + true, publicVlanId); } return ipToReturn; @@ -1223,6 +1301,23 @@ public PublicIp assignSourceNatIpAddressToGuestNetwork(Account owner, Network gu @Override public PublicIp assignDedicateIpAddress(Account owner, final Long guestNtwkId, final Long vpcId, final long dcId, final boolean isSourceNat) throws ConcurrentOperationException, InsufficientAddressCapacityException { + return assignDedicateIpAddress(owner, guestNtwkId, vpcId, dcId, isSourceNat, null, false); + } + + @DB + @Override + public PublicIp assignDedicateIpAddressFromNsxVrfPublicRange(Account owner, final Long guestNtwkId, + final Long vpcId, final long dcId, final boolean isSourceNat, final Long vlanDbId) + throws ConcurrentOperationException, InsufficientAddressCapacityException { + validateRegisteredNsxVrfPublicVlan(dcId, vlanDbId); + return assignDedicateIpAddress(owner, guestNtwkId, vpcId, dcId, isSourceNat, vlanDbId, true); + } + + @DB + private PublicIp assignDedicateIpAddress(Account owner, final Long guestNtwkId, final Long vpcId, + final long dcId, final boolean isSourceNat, final Long vlanDbId, + final boolean allowRegisteredNsxVrfVlan) + throws ConcurrentOperationException, InsufficientAddressCapacityException { final long ownerId = owner.getId(); @@ -1242,22 +1337,34 @@ public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAdd if (logger.isDebugEnabled()) { logger.debug("lock account {} is acquired", owner); } - List vlanDbIds = null; + List vlanDbIds = vlanDbId == null ? null : List.of(vlanDbId); boolean displayIp = true; if (guestNtwkId != null) { Network ntwk = _networksDao.findById(guestNtwkId); if (_networkOfferingDao.isIpv6Supported(ntwk.getNetworkOfferingId())) { - vlanDbIds = getIpv6SupportingVlanRangeIds(dcId); + List ipv6VlanDbIds = getIpv6SupportingVlanRangeIds(dcId); + if (vlanDbId != null && !ipv6VlanDbIds.contains(vlanDbId)) { + throw new InsufficientAddressCapacityException( + "The requested public IP range does not support IPv6", DataCenter.class, dcId); + } + vlanDbIds = vlanDbId == null ? ipv6VlanDbIds : vlanDbIds; } displayIp = ntwk.getDisplayNetwork(); } else if (vpcId != null) { VpcVO vpc = _vpcDao.findById(vpcId); if (vpcOfferingDao.isIpv6Supported(vpc.getVpcOfferingId())) { - vlanDbIds = getIpv6SupportingVlanRangeIds(dcId); + List ipv6VlanDbIds = getIpv6SupportingVlanRangeIds(dcId); + if (vlanDbId != null && !ipv6VlanDbIds.contains(vlanDbId)) { + throw new InsufficientAddressCapacityException( + "The requested public IP range does not support IPv6", DataCenter.class, dcId); + } + vlanDbIds = vlanDbId == null ? ipv6VlanDbIds : vlanDbIds; } displayIp = vpc.isDisplay(); } - return fetchNewPublicIp(dcId, null, vlanDbIds, owner, VlanType.VirtualNetwork, guestNtwkId, isSourceNat, true, null, null, false, vpcId, displayIp, false); + return fetchNewPublicIp(dcId, null, vlanDbIds, owner, VlanType.VirtualNetwork, guestNtwkId, + isSourceNat, true, true, null, null, false, vpcId, displayIp, false, + vlanDbId == null, allowRegisteredNsxVrfVlan); } }); if (ip.getState() != State.Allocated) { @@ -1447,10 +1554,36 @@ private void checkPublicIpOnExternalProviderZone(DataCenter zone, String ip) { } } + private void validateRegisteredNsxVrfPublicVlan(long zoneId, Long vlanDbId) { + NsxVrfGatewayVO gateway = vlanDbId == null ? null : nsxVrfGatewayDao.findByPublicVlan(vlanDbId); + if (gateway == null || gateway.getZoneId() != zoneId) { + throw new InvalidParameterValueException( + "The requested public IP range is not registered to an NSX VRF gateway in this zone"); + } + } + @DB @Override public IpAddress allocateIp(final Account ipOwner, final boolean isSystem, Account caller, User callerUser, final DataCenter zone, final Boolean displayIp, final String ipaddress) throws ConcurrentOperationException, InsufficientAddressCapacityException, CloudRuntimeException { + return allocateIp(ipOwner, isSystem, caller, callerUser, zone, displayIp, ipaddress, null, false); + } + + @DB + @Override + public IpAddress allocateIpFromNsxVrfPublicRange(final Account ipOwner, final boolean isSystem, + Account caller, User callerUser, final DataCenter zone, final Boolean displayIp, + final String ipaddress, final Long vlanDbId) + throws ConcurrentOperationException, InsufficientAddressCapacityException, CloudRuntimeException { + validateRegisteredNsxVrfPublicVlan(zone.getId(), vlanDbId); + return allocateIp(ipOwner, isSystem, caller, callerUser, zone, displayIp, ipaddress, vlanDbId, true); + } + + @DB + private IpAddress allocateIp(final Account ipOwner, final boolean isSystem, Account caller, User callerUser, + final DataCenter zone, final Boolean displayIp, final String ipaddress, final Long vlanDbId, + final boolean allowRegisteredNsxVrfVlan) + throws ConcurrentOperationException, InsufficientAddressCapacityException, CloudRuntimeException { final VlanType vlanType = VlanType.VirtualNetwork; final boolean assign = false; @@ -1484,6 +1617,14 @@ public IpAddress allocateIp(final Account ipOwner, final boolean isSystem, Accou if (ipaddress != null) { IPAddressVO ipAddr = _ipAddressDao.findByAccountIdAndZoneIdAndStateAndIpAddress(ipOwner.getId(), zone.getId(), State.Reserved, ipaddress); if (ipAddr != null) { + if (!allowRegisteredNsxVrfVlan && nsxVrfGatewayDao.findByPublicVlan(ipAddr.getVlanId()) != null) { + throw new InvalidParameterValueException( + "The requested public IP belongs to a range registered to an NSX VRF gateway"); + } + if (vlanDbId != null && !Objects.equals(vlanDbId, ipAddr.getVlanId())) { + throw new InvalidParameterValueException( + "The requested public IP is not in the required public IP range"); + } return PublicIp.createFromAddrAndVlan(ipAddr, _vlanDao.findById(ipAddr.getVlanId())); } } @@ -1491,7 +1632,10 @@ public IpAddress allocateIp(final Account ipOwner, final boolean isSystem, Accou ip = Transaction.execute(new TransactionCallbackWithException() { @Override public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException { - PublicIp ip = fetchNewPublicIp(zone.getId(), null, null, ipOwner, vlanType, null, false, assign, ipaddress, null, isSystem, null, displayIp, false); + List vlanDbIds = vlanDbId == null ? null : List.of(vlanDbId); + PublicIp ip = fetchNewPublicIp(zone.getId(), null, vlanDbIds, ipOwner, vlanType, null, + false, true, assign, ipaddress, null, isSystem, null, displayIp, false, + vlanDbId == null, allowRegisteredNsxVrfVlan); if (ip == null) { InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Unable to find available public IP addresses", DataCenter.class, zone diff --git a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java index a09867b8ffc3..6e2193f2c23e 100644 --- a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java @@ -232,6 +232,7 @@ public class NetworkMigrationManagerImpl implements NetworkMigrationManager { copiedNetwork.setDisplayNetwork(false); copiedNetwork.setBroadcastUri(network.getBroadcastUri()); copiedNetwork.setState(network.getState()); + copiedNetwork.setNetworkACLId(network.getNetworkACLId()); _networksDao.update(networkCopyId, copiedNetwork); copyNetworkDetails(originalNetworkId, networkCopyId); diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2853fa96330d..b0ec0bd3fcce 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -165,6 +165,7 @@ import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; import com.cloud.network.dao.OvsProviderDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; @@ -430,6 +431,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C @Inject NsxProviderDao nsxProviderDao; @Inject + NsxVrfGatewayDao nsxVrfGatewayDao; + @Inject private VirtualRouterProviderDao virtualRouterProviderDao; @Inject RoutedIpv4Manager routedIpv4Manager; @@ -731,16 +734,43 @@ public List getIsolatedNetworksWithSourceNATOwnedByAccountInZ @ActionEvent(eventType = EventTypes.EVENT_NET_IP_ASSIGN, eventDescription = "allocating Ip", create = true) public IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Boolean displayIp, String ipaddress) throws ResourceAllocationException, InsufficientAddressCapacityException, ConcurrentOperationException { + return allocateIP(ipOwner, zoneId, networkId, null, displayIp, ipaddress); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_NET_IP_ASSIGN, eventDescription = "allocating Ip", create = true) + public IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Long vpcId, Boolean displayIp, + String ipaddress) + throws ResourceAllocationException, InsufficientAddressCapacityException, ConcurrentOperationException { Account caller = CallContext.current().getCallingAccount(); User callerUser = CallContext.current().getCallingUser(); DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); + Long publicVlanId = null; + VpcVO nsxVpc = null; + Network nsxNetwork = null; + + if (vpcId != null) { + VpcVO vpc = _vpcDao.findById(vpcId); + if (vpc == null || vpc.getZoneId() != zoneId || vpc.getAccountId() != ipOwner.getId()) { + throw new InvalidParameterValueException("The VPC does not belong to the requested account and zone"); + } + if (isNsxTier1Vpc(vpc)) { + nsxVpc = vpc; + } + } if (networkId != null) { Network network = _networksDao.findById(networkId); if (network == null) { throw new InvalidParameterValueException("Invalid network id is given"); } + if (network.getDataCenterId() != zoneId || network.getAccountId() != ipOwner.getId()) { + throw new InvalidParameterValueException("The network does not belong to the requested account and zone"); + } + if (network.getBroadcastDomainType() == BroadcastDomainType.NSX) { + nsxNetwork = network; + } if (network.getGuestType() == Network.GuestType.Shared) { if (zone == null) { @@ -764,13 +794,40 @@ public IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Boolea _accountMgr.checkAccess(caller, null, false, ipOwner); } - IpAddress address = _ipAddrMgr.allocateIp(ipOwner, false, caller, callerUser, zone, displayIp, ipaddress); + if (nsxVpc != null || nsxNetwork != null) { + NsxService nsxService = ComponentContext.getDelegateComponentOfType(NsxService.class); + if (nsxService == null) { + throw new CloudRuntimeException("The NSX service is unavailable"); + } + if (nsxVpc != null) { + publicVlanId = nsxService.reserveTier1PlacementAndGetPublicVlanId(zoneId, + nsxVpc.getAccountId(), nsxVpc.getDomainId(), nsxVpc.getId(), null); + } else { + Long networkVpcId = nsxNetwork.getVpcId(); + publicVlanId = nsxService.reserveTier1PlacementAndGetPublicVlanId(zoneId, + nsxNetwork.getAccountId(), nsxNetwork.getDomainId(), networkVpcId, + networkVpcId == null ? nsxNetwork.getId() : null); + } + } + + IpAddress address = publicVlanId == null + ? _ipAddrMgr.allocateIp(ipOwner, false, caller, callerUser, zone, displayIp, ipaddress) + : _ipAddrMgr.allocateIpFromNsxVrfPublicRange(ipOwner, false, caller, callerUser, zone, + displayIp, ipaddress, publicVlanId); if (address != null) { CallContext.current().putContextParameter(IpAddress.class, address.getUuid()); } return address; } + private boolean isNsxTier1Vpc(Vpc vpc) { + Map> providers = _vpcMgr.getVpcOffSvcProvidersMap(vpc.getVpcOfferingId()); + Set sourceNatProviders = providers.get(Service.SourceNat); + Set gatewayProviders = providers.get(Service.Gateway); + return (sourceNatProviders != null && sourceNatProviders.contains(Provider.Nsx)) + || (gatewayProviders != null && gatewayProviders.contains(Provider.Nsx)); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_PORTABLE_IP_ASSIGN, eventDescription = "allocating portable public Ip", create = true) public IpAddress allocatePortableIP(Account ipOwner, int regionId, Long zoneId, Long networkId, Long vpcId) @@ -814,6 +871,10 @@ public IpAddress allocatePortableIP(Account ipOwner, int regionId, Long zoneId, if (vpc == null) { throw new InvalidParameterValueException("Invalid vpc id is given"); } + if (isNsxTier1Vpc(vpc)) { + throw new InvalidParameterValueException( + "Portable public IP ranges are not supported for NSX Tier-1 VPCs"); + } } _accountMgr.checkAccess(caller, null, false, ipOwner); @@ -1138,6 +1199,10 @@ public IpAddress reserveIpAddress(Account account, Boolean displayIp, Long ipAdd if (!vlan.getVlanType().equals(VlanType.VirtualNetwork)) { throw new IllegalArgumentException("Only IP addresses that belong to a virtual network may be reserved."); } + if (nsxVrfGatewayDao.findByPublicVlan(vlan.getId()) != null) { + throw new InvalidParameterValueException( + "Unable to reserve an IP from a range registered to an NSX VRF gateway"); + } if (ipVO.isPortable()) { throw new InvalidParameterValueException("Unable to reserve a portable IP."); } @@ -1224,6 +1289,9 @@ public IpAddress reserveIpAddressWithVlanDetail(Account account, DataCenter zone private VlanVO findOneVlanRangeMatchingVlanDetailKey(DataCenter zone, String vlanDetailKey) { List zoneVlans = _vlanDao.listByZone(zone.getId()); for (VlanVO zoneVlan : zoneVlans) { + if (nsxVrfGatewayDao.findByPublicVlan(zoneVlan.getId()) != null) { + continue; + } VlanDetailsVO detail = vlanDetailsDao.findDetail(zoneVlan.getId(), vlanDetailKey); if (detail != null && detail.getValue().equalsIgnoreCase("true")) { logger.debug(String.format("Found the VLAN range %s is set for NSX on zone %s", zoneVlan.getIpRange(), zone.getName())); @@ -1953,6 +2021,16 @@ private boolean checkAndUpdateRouterSourceNatIp(UpdateNetworkCmd cmd, Network ne List userIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), true); if (! userIps.isEmpty()) { try { + if (network.getBroadcastDomainType() == BroadcastDomainType.NSX) { + NsxService nsxService = ComponentContext.getDelegateComponentOfType(NsxService.class); + if (nsxService == null) { + throw new CloudRuntimeException("The NSX service is unavailable"); + } + Long vpcId = network.getVpcId(); + nsxService.validatePublicIpVlan(network.getDataCenterId(), network.getAccountId(), + network.getDomainId(), vpcId, vpcId == null ? network.getId() : null, + requestedIp.getVlanId()); + } _ipAddrMgr.updateSourceNatIpAddress(requestedIp, userIps); } catch (Exception e) { // pokemon exception from transaction String msg = String.format("Update of source NAT IP to %s for Network \"%s\"/%s failed due to %s", @@ -4194,6 +4272,12 @@ protected boolean canUpgrade(Network network, long oldNetworkOfferingId, long ne NetworkOffering oldNetworkOffering = _networkOfferingDao.findByIdIncludingRemoved(oldNetworkOfferingId); NetworkOffering newNetworkOffering = _networkOfferingDao.findById(newNetworkOfferingId); + if (!haveMatchingNsxSegmentProfiles(oldNetworkOfferingId, newNetworkOfferingId)) { + logger.debug("Network offerings {} and {} have different NSX segment profile bindings, can't upgrade", + oldNetworkOffering, newNetworkOffering); + return false; + } + // security group service should be the same if (areServicesSupportedByNetworkOffering(oldNetworkOfferingId, Service.SecurityGroup) != areServicesSupportedByNetworkOffering(newNetworkOfferingId, Service.SecurityGroup)) { logger.debug("Offerings {} and {} have different securityGroupProperty, can't upgrade", newNetworkOffering, oldNetworkOffering); @@ -4230,6 +4314,20 @@ protected boolean canUpgrade(Network network, long oldNetworkOfferingId, long ne return canMoveToPhysicalNetwork(network, oldNetworkOfferingId, newNetworkOfferingId); } + protected boolean haveMatchingNsxSegmentProfiles(long oldNetworkOfferingId, long newNetworkOfferingId) { + Map oldDetails = _networkModel.getNtwkOffDetails(oldNetworkOfferingId); + Map newDetails = _networkModel.getNtwkOffDetails(newNetworkOfferingId); + for (NetworkOffering.Detail detail : List.of(NetworkOffering.Detail.NsxIpDiscoveryProfileId, + NetworkOffering.Detail.NsxMacDiscoveryProfileId, NetworkOffering.Detail.NsxSegmentSecurityProfileId)) { + String oldProfileId = oldDetails == null ? null : oldDetails.get(detail); + String newProfileId = newDetails == null ? null : newDetails.get(detail); + if (!Objects.equals(oldProfileId, newProfileId)) { + return false; + } + } + return true; + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_PHYSICAL_NETWORK_CREATE, eventDescription = "Creating Physical Network", create = true) diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index 49c2243d1ecf..a9575380df32 100644 --- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -370,7 +371,9 @@ private void appendSourceNatIpToBootArgs(final VirtualMachineProfile profile) { final StringBuilder buf = profile.getBootArgsBuilder(); final DomainRouterVO router = _routerDao.findById(profile.getVirtualMachine().getId()); if (router != null && router.getVpcId() != null) { - List vpcIps = _ipAddressDao.listByAssociatedVpc(router.getVpcId(), true); + List vpcIps = _ipAddressDao.listByAssociatedVpc(router.getVpcId(), true).stream() + .filter(ip -> !ip.isForSystemVms()) + .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(vpcIps)) { buf.append(String.format(" source_nat_ip=%s", vpcIps.get(0).getAddress().toString())); logger.debug("The final Boot Args for " + profile + ": " + buf); diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java index bcf2c6176efe..87ad86971a49 100644 --- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java @@ -36,6 +36,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.cloud.event.UsageEventUtils; import javax.annotation.PostConstruct; @@ -151,6 +152,7 @@ import com.cloud.network.element.NsxProviderVO; import com.cloud.network.element.StaticNatServiceProvider; import com.cloud.network.element.VpcProvider; +import com.cloud.network.nsx.NsxService; import com.cloud.network.router.CommandSetupHelper; import com.cloud.network.router.NetworkHelper; import com.cloud.network.router.VpcVirtualNetworkApplianceManager; @@ -187,6 +189,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; +import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -223,6 +226,10 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis public static final String CAPABILITYVALUE = "capabilityvalue"; public static final String TRUE_VALUE = "true"; public static final String FALSE_VALUE = "false"; + protected static final String DEFAULT_ALLOW_ACL_NAME = "default_allow"; + protected static final String DEFAULT_DENY_ACL_NAME = "default_deny"; + protected static final String KUBERNETES_SERVICE_NETWORK_OFFERING_NAME_TOKEN = "KubernetesService"; + protected static final String KUBERNETES_CLUSTER_NETWORK_OFFERING_CONFIG_KEY = "cloud.kubernetes.cluster.network.offering"; @Inject EntityManager _entityMgr; @@ -1743,8 +1750,8 @@ public Vpc createVpc(CreateVPCCmd cmd) throws ResourceAllocationException { cmd.getUseVrIpResolver(), cmd.getKeepMacAddressOnPublicNic()); String sourceNatIP = cmd.getSourceNatIP(); - boolean forNsx = isVpcForProvider(Provider.Nsx, vpc); - boolean forNetris = isVpcForProvider(Provider.Netris, vpc); + boolean forNsx = isVpcTier1ForProvider(Provider.Nsx, vpc); + boolean forNetris = isVpcTier1ForProvider(Provider.Netris, vpc); try { if (sourceNatIP != null || forNsx || forNetris) { if (forNsx || forNetris) { @@ -1784,15 +1791,32 @@ private boolean isVpcForProvider(Provider provider, Vpc vpc) { return _vpcOffSvcMapDao.isProviderForVpcOffering(provider, vpc.getVpcOfferingId()); } + private boolean isVpcTier1ForProvider(Provider provider, Vpc vpc) { + if (vpc == null) { + return false; + } + Map> serviceProviders = getVpcOffSvcProvidersMap(vpc.getVpcOfferingId()); + Set sourceNatProviders = serviceProviders.get(Service.SourceNat); + Set gatewayProviders = serviceProviders.get(Service.Gateway); + return (sourceNatProviders != null && sourceNatProviders.contains(provider)) + || (gatewayProviders != null && gatewayProviders.contains(provider)); + } + private void allocateSourceNatIp(Vpc vpc, String sourceNatIP) { Account account = _accountMgr.getAccount(vpc.getAccountId()); DataCenter zone = _dcDao.findById(vpc.getZoneId()); // reserve this ip and then try { - if (isVpcForProvider(Provider.Nsx, vpc) && org.apache.commons.lang3.StringUtils.isBlank(sourceNatIP)) { - logger.debug(String.format("Reserving a source NAT IP for NSX VPC %s", vpc.getName())); + if (isVpcTier1ForProvider(Provider.Nsx, vpc) && org.apache.commons.lang3.StringUtils.isBlank(sourceNatIP)) { + Long vrfPublicVlanId = getNsxPublicVlanId(vpc); + if (vrfPublicVlanId != null) { + _ipAddrMgr.assignDedicateIpAddressFromNsxVrfPublicRange(account, null, vpc.getId(), + zone.getId(), true, vrfPublicVlanId); + return; + } + logger.debug("Reserving a source NAT IP for NSX VPC {}", vpc.getName()); sourceNatIP = reserveSourceNatIpForProviderVpc(account, zone, Provider.Nsx); - } else if (isVpcForProvider(Provider.Netris, vpc) && org.apache.commons.lang3.StringUtils.isBlank(sourceNatIP)) { + } else if (isVpcTier1ForProvider(Provider.Netris, vpc) && org.apache.commons.lang3.StringUtils.isBlank(sourceNatIP)) { logger.debug(String.format("Reserving a source NAT IP for Netris VPC %s", vpc.getName())); sourceNatIP = reserveSourceNatIpForProviderVpc(account, zone, Provider.Netris); } @@ -1803,6 +1827,15 @@ private void allocateSourceNatIp(Vpc vpc, String sourceNatIP) { } } + private Long getNsxPublicVlanId(Vpc vpc) { + NsxService nsxService = ComponentContext.getDelegateComponentOfType(NsxService.class); + if (nsxService == null) { + throw new CloudRuntimeException("The NSX service is unavailable"); + } + return nsxService.reserveTier1PlacementAndGetPublicVlanId(vpc.getZoneId(), vpc.getAccountId(), vpc.getDomainId(), + vpc.getId(), null); + } + private String reserveSourceNatIpForProviderVpc(Account account, DataCenter zone, Provider provider) throws ResourceAllocationException { String detailKey = provider == Provider.Nsx ? ApiConstants.NSX_DETAIL_KEY : ApiConstants.NETRIS_DETAIL_KEY; IpAddress ipAddress = _ntwkSvc.reserveIpAddressWithVlanDetail(account, zone, true, detailKey); @@ -2047,16 +2080,16 @@ private boolean checkAndUpdateRouterSourceNatIp(Vpc vpc, String sourceNatIp) { IPAddressVO requestedIp = validateSourceNatip(vpc, sourceNatIp); if (requestedIp == null) return false; // ip not associated with this network - List userIps = _ipAddressDao.listByAssociatedVpc(vpc.getId(), true); + List userIps = _ipAddressDao.listByAssociatedVpc(vpc.getId(), true).stream() + .filter(ip -> !ip.isForSystemVms()) + .collect(Collectors.toList()); if (! userIps.isEmpty()) { try { + validateNsxPublicIpVlan(vpc, requestedIp); _ipAddrMgr.updateSourceNatIpAddress(requestedIp, userIps); - if (isVpcForProvider(Provider.Nsx, vpc) || isVpcForProvider(Provider.Netris, vpc) - || isVpcForProvider(Provider.NetworkExtension, vpc)) { - boolean isForNsx = _vpcOffSvcMapDao.isProviderForVpcOffering(Provider.Nsx, vpc.getVpcOfferingId()); - boolean isForNetris = _vpcOffSvcMapDao.isProviderForVpcOffering(Provider.Netris, vpc.getVpcOfferingId()); - String providerName = isForNsx ? Provider.Nsx.getName() - : (isForNetris ? Provider.Netris.getName() : Provider.NetworkExtension.getName()); + String providerName = getVpcTier1ProviderName(vpc); + if (List.of(Provider.Nsx.getName(), Provider.Netris.getName(), Provider.NetworkExtension.getName()) + .contains(providerName)) { VpcProvider providerElement = (VpcProvider) _ntwkModel.getElementImplementingProvider(providerName); if (Objects.nonNull(providerElement)) { providerElement.updateVpcSourceNatIp(vpc, requestedIp); @@ -2074,6 +2107,22 @@ private boolean checkAndUpdateRouterSourceNatIp(Vpc vpc, String sourceNatIp) { return true; } + private String getVpcTier1ProviderName(Vpc vpc) { + String provider = _vpcSrvcDao.getProviderForServiceInVpc(vpc.getId(), Service.SourceNat); + return provider == null ? _vpcSrvcDao.getProviderForServiceInVpc(vpc.getId(), Service.Gateway) : provider; + } + + private void validateNsxPublicIpVlan(Vpc vpc, IpAddress address) { + if (!isVpcTier1ForProvider(Provider.Nsx, vpc)) { + return; + } + Long expectedVlanId = getNsxPublicVlanId(vpc); + if (expectedVlanId != null && !Objects.equals(expectedVlanId, address.getVlanId())) { + throw new InvalidParameterValueException( + "The public IP is not from the range registered to the VPC's NSX VRF gateway"); + } + } + @Nullable protected IPAddressVO validateSourceNatip(Vpc vpc, String sourceNatIp) { if (sourceNatIp == null) { @@ -3721,7 +3770,8 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ VpcTierNamePrepend, - VpcTierNamePrependDelimiter + VpcTierNamePrependDelimiter, + VpcTierDefaultNetworkACL }; } @@ -3786,6 +3836,8 @@ public IpAddress associateIPToVpc(final long ipId, final long vpcId) throws Reso // check permissions _accountMgr.checkAccess(caller, null, false, owner, vpc); + validateNsxPublicIpVlan(vpc, ipToAssoc); + logger.debug(String.format("Associating IP [%s] to VPC [%s]", ipToAssoc, vpc)); final boolean isSourceNatFinal = isSrcNatIpRequired(vpc.getVpcOfferingId()) && getExistingSourceNatInVpc(vpc.getAccountId(), vpcId, false, false) == null; @@ -3852,6 +3904,24 @@ public boolean isIpAllocatedToVpc(final IpAddress ip) { return ip != null && ip.getVpcId() != null && (ip.isOneToOneNat() || !_firewallDao.listByIp(ip.getId()).isEmpty()); } + /** + * Resolves the ACL a new VPC tier gets when the caller does not specify one. Kubernetes clusters + * cannot be provisioned on a tier that uses the default deny ACL, and default ACLs cannot be given + * rules, so tiers on the offerings the Kubernetes service deploys into always default to the + * default allow ACL; every other tier follows the VpcTierDefaultNetworkACL setting. + */ + protected Long getDefaultAclIdForNewTier(long ntwkOffId, long zoneId) { + NetworkOffering networkOffering = _entityMgr.findById(NetworkOffering.class, ntwkOffId); + String offeringName = networkOffering == null ? null : networkOffering.getName(); + if (org.apache.commons.lang3.StringUtils.isNotBlank(offeringName) + && (offeringName.contains(KUBERNETES_SERVICE_NETWORK_OFFERING_NAME_TOKEN) + || offeringName.equalsIgnoreCase(_configDao.getValue(KUBERNETES_CLUSTER_NETWORK_OFFERING_CONFIG_KEY)))) { + return NetworkACL.DEFAULT_ALLOW; + } + return DEFAULT_DENY_ACL_NAME.equalsIgnoreCase(VpcTierDefaultNetworkACL.valueIn(zoneId)) + ? NetworkACL.DEFAULT_DENY : NetworkACL.DEFAULT_ALLOW; + } + @DB @Override public Network createVpcGuestNetwork(final long ntwkOffId, final String name, final String displayText, final String gateway, final String cidr, final String vlanId, @@ -3880,12 +3950,19 @@ public Network createVpcGuestNetwork(final long ntwkOffId, final String name, fi // 1) Validate if network can be created for VPC validateNtwkOffForNtwkInVpc(null, ntwkOffId, cidr, networkDomain, vpc, gateway, owner, aclId); + Long effectiveAclId = aclId; + if (effectiveAclId == null && _ntwkModel.areServicesSupportedByNetworkOffering(ntwkOffId, Service.NetworkACL)) { + effectiveAclId = getDefaultAclIdForNewTier(ntwkOffId, zoneId); + logger.debug("No ACL provided for the new tier in VPC {}, defaulting to the {} ACL", vpc, + NetworkACL.DEFAULT_ALLOW == effectiveAclId ? DEFAULT_ALLOW_ACL_NAME : DEFAULT_DENY_ACL_NAME); + } + // 2) Create network final Network guestNetwork = _ntwkMgr.createGuestNetwork(ntwkOffId, name, displayText, gateway, cidr, vlanId, false, networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, isDisplayNetworkEnabled, null, null, externalId, null, null, ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize); if (guestNetwork != null) { - guestNetwork.setNetworkACLId(aclId); + guestNetwork.setNetworkACLId(effectiveAclId); _ntwkDao.update(guestNetwork.getId(), (NetworkVO) guestNetwork); } return guestNetwork; diff --git a/server/src/main/java/com/cloud/network/vpn/RemoteAccessVpnManagerImpl.java b/server/src/main/java/com/cloud/network/vpn/RemoteAccessVpnManagerImpl.java index 24c2f2221d96..d7d650f27781 100644 --- a/server/src/main/java/com/cloud/network/vpn/RemoteAccessVpnManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpn/RemoteAccessVpnManagerImpl.java @@ -60,6 +60,7 @@ import com.cloud.network.dao.RemoteAccessVpnDao; import com.cloud.network.dao.RemoteAccessVpnVO; import com.cloud.network.dao.VpnUserDao; +import com.cloud.network.element.NetworkElement; import com.cloud.network.element.RemoteAccessVPNServiceProvider; import com.cloud.network.rules.FirewallManager; import com.cloud.network.rules.FirewallRule; @@ -282,8 +283,41 @@ public RemoteAccessVpn createRemoteAccessVpn(final long publicIpId, String ipRan } } - private void validateIpAddressForVpnServiceOnNetwork(Network network, IPAddressVO ipAddress) { + private RemoteAccessVPNServiceProvider getRemoteAccessVpnServiceProvider(Long networkId, Long vpcId) { + List providers = new ArrayList<>(); + for (RemoteAccessVPNServiceProvider provider : _vpnServiceProviders) { + if (!(provider instanceof NetworkElement)) { + continue; + } + Network.Provider networkProvider = ((NetworkElement) provider).getProvider(); + boolean supportsRemoteAccessVpn = vpcId != null + ? vpcManager.isProviderSupportServiceInVpc(vpcId, Service.Vpn, networkProvider) + : networkId != null && _networkMgr.isProviderSupportServiceInNetwork(networkId, Service.Vpn, networkProvider); + if (supportsRemoteAccessVpn) { + providers.add(provider); + } + } + if (providers.size() > 1) { + throw new InvalidParameterValueException(String.format( + "More than one Remote Access VPN provider is configured for %s %s", + vpcId != null ? "VPC" : "network", vpcId != null ? vpcId : networkId)); + } + return providers.isEmpty() ? null : providers.get(0); + } + + private RemoteAccessVPNServiceProvider requireRemoteAccessVpnServiceProvider(Long networkId, Long vpcId) { + RemoteAccessVPNServiceProvider provider = getRemoteAccessVpnServiceProvider(networkId, vpcId); + if (provider == null) { + throw new InvalidParameterValueException(String.format( + "Remote Access VPN is not supported for %s %s because its configured Vpn provider does not implement the Remote Access VPN service", + vpcId != null ? "VPC" : "network", vpcId != null ? vpcId : networkId)); + } + return provider; + } + + void validateIpAddressForVpnServiceOnNetwork(Network network, IPAddressVO ipAddress) { Long networkId = network.getId(); + requireRemoteAccessVpnServiceProvider(networkId, null); if (_networkMgr.isProviderSupportServiceInNetwork(networkId, Service.Vpn, Network.Provider.VirtualRouter)) { // if VR is the VPN provider, // (1) if VR is Source NAT, the IP address must be used as Source NAT @@ -301,8 +335,9 @@ private void validateIpAddressForVpnServiceOnNetwork(Network network, IPAddressV } } - private void validateIpAddressForVpnServiceOnVpc(Vpc vpc, IPAddressVO ipAddress) { + void validateIpAddressForVpnServiceOnVpc(Vpc vpc, IPAddressVO ipAddress) { Long vpcId = vpc.getId(); + requireRemoteAccessVpnServiceProvider(null, vpcId); if (vpcManager.isProviderSupportServiceInVpc(vpcId, Service.Vpn, Network.Provider.VPCVirtualRouter)) { // if VPC VR is the VPN provider, // (1) if VPC VR is Source NAT, the IP address must be used as Source NAT @@ -545,6 +580,7 @@ public RemoteAccessVpnVO startRemoteAccessVpn(long ipAddressId, boolean openFire _accountMgr.checkAccess(caller, null, true, vpn); + RemoteAccessVPNServiceProvider provider = requireRemoteAccessVpnServiceProvider(vpn.getNetworkId(), vpn.getVpcId()); boolean started = false; try { boolean firewallOpened = true; @@ -552,13 +588,13 @@ public RemoteAccessVpnVO startRemoteAccessVpn(long ipAddressId, boolean openFire firewallOpened = _firewallMgr.applyIngressFirewallRules(vpn.getServerAddressId(), caller); } - if (firewallOpened) { - for (RemoteAccessVPNServiceProvider element : _vpnServiceProviders) { - if (element.startVpn(vpn)) { - started = true; - break; - } - } + if (firewallOpened && provider.startVpn(vpn)) { + started = true; + } + if (!started) { + throw new ResourceUnavailableException(String.format( + "Failed to start Remote Access VPN %s using provider %s", vpn.getId(), provider.getName()), + RemoteAccessVpn.class, vpn.getId()); } return vpn; diff --git a/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java b/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java index 47362feb4d15..9912ae7470ff 100644 --- a/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java @@ -24,6 +24,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -62,6 +63,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; import com.cloud.network.Site2SiteCustomerGateway; @@ -76,6 +78,7 @@ import com.cloud.network.dao.Site2SiteVpnConnectionVO; import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.Site2SiteVpnGatewayVO; +import com.cloud.network.element.NetworkElement; import com.cloud.network.element.Site2SiteVpnServiceProvider; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcManager; @@ -214,36 +217,137 @@ public Site2SiteVpnGateway createVpnGateway(CreateVpnGatewayCmd cmd) { throw new InvalidParameterValueException(String.format("The VPN gateway of VPC %s already exists!", vpc)); } - IPAddressVO requestedIp = _ipAddressDao.findById(cmd.getIpAddressId()); - IPAddressVO ipAddress = getIpAddressIdForVpn(vpcId, vpc.getVpcOfferingId(), requestedIp); + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForVpc(vpcId); + if (provider == null) { + throw new InvalidParameterValueException(String.format( + "Site-to-Site VPN is not supported by %s: the VPC offering does not provide the Vpn service through any available VPN provider", + vpc)); + } + + Long requestedIpId = cmd.getIpAddressId(); + IPAddressVO requestedIp = requestedIpId == null ? null : _ipAddressDao.findById(requestedIpId); + if (requestedIpId != null && requestedIp == null) { + throw new InvalidParameterValueException(String.format( + "Unable to find the requested VPN gateway IP with id %s", requestedIpId)); + } + IPAddressVO ipAddress = getIpAddressIdForVpn(vpc, provider, requestedIp); Site2SiteVpnGatewayVO gw = new Site2SiteVpnGatewayVO(owner.getAccountId(), owner.getDomainId(), ipAddress.getId(), vpcId); if (cmd.getDisplay() != null) { gw.setDisplay(cmd.getDisplay()); } - _vpnGatewayDao.persist(gw); + try { + _vpnGatewayDao.persist(gw); + } catch (RuntimeException e) { + try { + provider.releaseVpnGatewayIp(gw); + } catch (Exception releaseException) { + logger.warn("Failed to release the VPN gateway resources of VPC {} after the gateway could not be persisted: {}", + vpc, releaseException.getMessage()); + } + throw e; + } return gw; } - private IPAddressVO getIpAddressIdForVpn(Long vpcId, Long vpcOferingId, IPAddressVO requestedIp) { - VpcOfferingServiceMapVO mapForSourceNat = vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.SourceNat.getName(), Network.Provider.VPCVirtualRouter.getName(), vpcOferingId); - VpcOfferingServiceMapVO mapForVpn = vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.Vpn.getName(), Network.Provider.VPCVirtualRouter.getName(), vpcOferingId); + private List getVpnServiceProvidersForVpc(long vpcId) { + List providers = new ArrayList<>(); + for (Site2SiteVpnServiceProvider provider : _s2sProviders) { + if (provider instanceof NetworkElement + && vpcManager.isProviderSupportServiceInVpc(vpcId, Network.Service.Vpn, + ((NetworkElement) provider).getProvider())) { + providers.add(provider); + } + } + return providers; + } + + private Site2SiteVpnServiceProvider getVpnServiceProviderForVpc(long vpcId) { + List providers = getVpnServiceProvidersForVpc(vpcId); + if (providers.size() > 1) { + throw new InvalidParameterValueException(String.format( + "VPC %s has more than one Site-to-Site VPN provider; exactly one provider must be configured", + vpcId)); + } + return providers.isEmpty() ? null : providers.get(0); + } + + private Site2SiteVpnServiceProvider getVpnServiceProviderForGateway(Site2SiteVpnGateway gateway) { + List ownedProviders = new ArrayList<>(); + for (Site2SiteVpnServiceProvider provider : _s2sProviders) { + if (provider.ownsVpnGateway(gateway)) { + ownedProviders.add(provider); + } + } + if (ownedProviders.size() > 1) { + throw new CloudRuntimeException(String.format( + "VPN gateway %s is claimed by more than one Site-to-Site VPN provider", gateway.getId())); + } + if (!ownedProviders.isEmpty()) { + return ownedProviders.get(0); + } + // Gateways created before provider ownership was persisted were all terminated by the + // VPC virtual router. Keep their lifecycle on that provider even if the offering changes. + for (Site2SiteVpnServiceProvider provider : _s2sProviders) { + if (provider instanceof NetworkElement + && Network.Provider.VPCVirtualRouter.equals(((NetworkElement) provider).getProvider())) { + return provider; + } + } + return null; + } + + private IPAddressVO getIpAddressIdForVpn(Vpc vpc, Site2SiteVpnServiceProvider provider, IPAddressVO requestedIp) { + IpAddress providerIp = provider.acquireVpnGatewayIp(vpc, requestedIp); + if (providerIp != null) { + logger.debug("Using VPN gateway IP {} supplied by provider {} for VPC {}", + providerIp.getAddress(), provider.getName(), vpc); + IPAddressVO providerIpRow = _ipAddressDao.findById(providerIp.getId()); + if (providerIpRow == null) { + releaseProviderVpnGatewayIp(provider, vpc, providerIp.getId()); + throw new CloudRuntimeException(String.format( + "VPN provider %s returned IP id %s for VPC %s, but the IP no longer exists", + provider.getName(), providerIp.getId(), vpc.getId())); + } + boolean addressMismatch = providerIp.getAddress() == null || providerIpRow.getAddress() == null + || !providerIp.getAddress().addr().equals(providerIpRow.getAddress().addr()); + if (providerIpRow.getRemoved() != null || !providerIpRow.readyToUse() + || providerIpRow.getVpcId() == null || providerIpRow.getVpcId() != vpc.getId() + || providerIpRow.isSourceNat() || providerIpRow.isForSystemVms() || addressMismatch) { + releaseProviderVpnGatewayIp(provider, vpc, providerIp.getId()); + throw new CloudRuntimeException(String.format( + "VPN provider %s returned IP id %s that is not an active, dedicated IP of VPC %s", + provider.getName(), providerIp.getId(), vpc.getId())); + } + return providerIpRow; + } + + VpcOfferingServiceMapVO mapForSourceNat = vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.SourceNat.getName(), Network.Provider.VPCVirtualRouter.getName(), vpc.getVpcOfferingId()); + VpcOfferingServiceMapVO mapForVpn = vpcOfferingServiceMapDao.findByServiceProviderAndOfferingId(Network.Service.Vpn.getName(), Network.Provider.VPCVirtualRouter.getName(), vpc.getVpcOfferingId()); if (mapForSourceNat == null && mapForVpn != null) { // Use Static NAT IP of VPC VR logger.debug(String.format("The VPC VR provides %s Service, however it does not provide %s service, trying to configure using IP of VPC VR", Network.Service.Vpn.getName(), Network.Service.SourceNat.getName())); - Vpc vpc = _vpcDao.findById(vpcId); IPAddressVO ipAddressForVpcVR = vpcManager.getIpAddressForVpcVr(vpc, requestedIp, true); if (!vpcManager.configStaticNatForVpcVr(vpc, ipAddressForVpcVR)) { throw new CloudRuntimeException("Failed to enable static nat for VPC VR as part of vpn gateway"); } return ipAddressForVpcVR; } else { - //Use source NAT ip for VPC - List ips = _ipAddressDao.listByAssociatedVpc(vpcId, true); - if (ips.size() != 1) { - throw new CloudRuntimeException("Cannot found source nat ip of vpc " + vpcId); + // The VR helper address can also be marked source NAT for redeploy idempotency. + // It is not the customer-facing source NAT address used by the legacy VPN provider. + List ips = _ipAddressDao.listByAssociatedVpc(vpc.getId(), true).stream() + .filter(ip -> !ip.isForSystemVms()) + .collect(Collectors.toList()); + if (ips.isEmpty()) { + throw new CloudRuntimeException("No source NAT IP found for VPC " + vpc.getId()); + } + if (ips.size() > 1) { + String addresses = ips.stream().map(ip -> ip.getAddress().addr()).collect(Collectors.joining(", ")); + throw new CloudRuntimeException(String.format( + "Multiple source NAT IPs (%s) found for VPC %s while exactly one was expected", + addresses, vpc.getId())); } if (requestedIp != null && requestedIp.getId() != ips.get(0).getId()) { throw new CloudRuntimeException(String.format("Cannot use requested IP %s as it is not the Source NAT IP", requestedIp.getAddress().addr())); @@ -252,6 +356,16 @@ private IPAddressVO getIpAddressIdForVpn(Long vpcId, Long vpcOferingId, IPAddres } } + private void releaseProviderVpnGatewayIp(Site2SiteVpnServiceProvider provider, Vpc vpc, long ipAddressId) { + try { + provider.releaseVpnGatewayIp(new Site2SiteVpnGatewayVO(vpc.getAccountId(), + vpc.getDomainId(), ipAddressId, vpc.getId())); + } catch (Exception cleanupException) { + logger.warn("Failed to clean up VPN provider resources for IP {} of VPC {} after provider {} returned an invalid address: {}", + ipAddressId, vpc.getId(), provider.getName(), cleanupException.getMessage()); + } + } + private void validateVpnCryptographicParameters(String ikePolicy, String espPolicy, String ikeVersion, Long domainId) { String excludedEncryption = VpnCustomerGatewayExcludedEncryptionAlgorithms.valueIn(domainId); String excludedHashing = VpnCustomerGatewayExcludedHashingAlgorithms.valueIn(domainId); @@ -478,6 +592,7 @@ public Site2SiteVpnConnection createVpnConnection(CreateVpnConnectionCmd cmd) { validateVpnConnectionOfTheRightAccount(customerGateway, vpnGateway); validateVpnConnectionDoesntExist(customerGateway, vpnGateway); validatePrerequisiteVpnGateway(vpnGateway); + validateCustomerGatewayForVpnGateway(customerGateway, vpnGateway); String[] cidrList = customerGateway.getGuestCidrList().split(","); @@ -559,6 +674,16 @@ private void validatePrerequisiteVpnGateway(Site2SiteVpnGateway vpnGateway) { } } + private void validateCustomerGatewayForVpnGateway(Site2SiteCustomerGateway customerGateway, + Site2SiteVpnGateway vpnGateway) { + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForGateway(vpnGateway); + if (provider == null) { + throw new InvalidParameterValueException(String.format( + "No Site-to-Site VPN provider owns gateway %s", vpnGateway.getId())); + } + provider.validateSite2SiteVpnCustomerGateway(customerGateway); + } + @Override @DB @ActionEvent(eventType = EventTypes.EVENT_S2S_VPN_CONNECTION_CREATE, eventDescription = "starting s2s vpn connection", async = true) @@ -568,43 +693,57 @@ public Site2SiteVpnConnection startVpnConnection(long id) throws ResourceUnavail throw new CloudRuntimeException("Unable to acquire lock for starting of VPN connection with ID " + id); } try { - if (conn.getState() != State.Pending && conn.getState() != State.Disconnected) { - throw new InvalidParameterValueException( + return startVpnConnectionLocked(conn); + } finally { + _vpnConnectionDao.releaseFromLockTable(conn.getId()); + } + } + + private Site2SiteVpnConnectionVO startVpnConnectionLocked(Site2SiteVpnConnectionVO conn) throws ResourceUnavailableException { + if (conn.getState() != State.Pending && conn.getState() != State.Disconnected) { + throw new InvalidParameterValueException( "Site to site VPN connection with specified connectionId not in correct state(pending or disconnected) to process!"); - } + } - conn.setState(State.Pending); - _vpnConnectionDao.persist(conn); + conn.setState(State.Pending); + _vpnConnectionDao.persist(conn); - final Site2SiteVpnGateway vpnGateway = _vpnGatewayDao.findById(conn.getVpnGatewayId()); + try { + Site2SiteVpnGateway vpnGateway = getVpnGatewayForConnection(conn); try { vpcManager.applyStaticRouteForVpcVpnIfNeeded(vpnGateway.getVpcId(), false); } catch (ResourceUnavailableException | CloudRuntimeException e) { - logger.error("Unable to apply static routes for vpc " + vpnGateway.getVpcId() + "as part of start of VPN connection, due to " + e.getMessage()); + logger.error("Unable to apply static routes for vpc {} as part of start of VPN connection, due to {}", + vpnGateway.getVpcId(), e.getMessage()); } - - boolean result = true; - for (Site2SiteVpnServiceProvider element : _s2sProviders) { - result = result & element.startSite2SiteVpn(conn); + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForGateway(vpnGateway); + if (provider == null) { + throw new InvalidParameterValueException(String.format( + "No Site-to-Site VPN provider owns gateway %s", vpnGateway.getId())); } - - if (result) { - if (conn.isPassive()) { - conn.setState(State.Disconnected); - } else { - conn.setState(State.Connecting); - } - _vpnConnectionDao.persist(conn); - return conn; + if (!provider.startSite2SiteVpn(conn)) { + throw new ResourceUnavailableException("Failed to apply site-to-site VPN", + Site2SiteVpnConnection.class, conn.getId()); } + conn.setState(conn.isPassive() ? State.Disconnected : State.Connecting); + _vpnConnectionDao.persist(conn); + return conn; + } catch (ResourceUnavailableException | RuntimeException e) { conn.setState(State.Error); _vpnConnectionDao.persist(conn); - throw new ResourceUnavailableException("Failed to apply site-to-site VPN", Site2SiteVpnConnection.class, id); - } finally { - _vpnConnectionDao.releaseFromLockTable(conn.getId()); + throw e; } } + private Site2SiteVpnGateway getVpnGatewayForConnection(Site2SiteVpnConnection conn) { + Site2SiteVpnGateway vpnGateway = _vpnGatewayDao.findById(conn.getVpnGatewayId()); + if (vpnGateway == null) { + throw new CloudRuntimeException(String.format("Unable to find VPN gateway %s for connection %s", + conn.getVpnGatewayId(), conn.getUuid())); + } + return vpnGateway; + } + @Override public Site2SiteVpnGateway getVpnGateway(Long vpnGatewayId) { return _vpnGatewayDao.findById(vpnGatewayId); @@ -643,6 +782,11 @@ protected void doDeleteVpnGateway(Site2SiteVpnGateway gw) { if (!CollectionUtils.isEmpty(conns)) { throw new InvalidParameterValueException(String.format("Unable to delete VPN gateway %s because there is still related VPN connections!", gw)); } + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForGateway(gw); + if (provider == null) { + throw new CloudRuntimeException(String.format("No Site-to-Site VPN provider owns gateway %s", gw.getId())); + } + provider.releaseVpnGatewayIp(gw); _vpnGatewayDao.remove(gw.getId()); } @@ -660,6 +804,7 @@ public boolean deleteVpnGateway(DeleteVpnGatewayCmd cmd) { } @Override + @DB @ActionEvent(eventType = EventTypes.EVENT_S2S_VPN_CUSTOMER_GATEWAY_UPDATE, eventDescription = "update s2s vpn customer gateway", create = true) public Site2SiteCustomerGateway updateCustomerGateway(UpdateVpnCustomerGatewayCmd cmd) { Account caller = CallContext.current().getCallingAccount(); @@ -736,6 +881,23 @@ public Site2SiteCustomerGateway updateCustomerGateway(UpdateVpnCustomerGatewayCm throw new InvalidParameterValueException("The customer gateway with name " + name + " already exists!"); } + String effectiveIkeVersion = ikeVersion == null ? gw.getIkeVersion() : ikeVersion; + Site2SiteCustomerGatewayVO proposedGateway = new Site2SiteCustomerGatewayVO(name, accountId, gw.getDomainId(), + gatewayIp, guestCidrList, ipsecPsk, ikePolicy, espPolicy, ikeLifetime, espLifetime, dpd, encap, + splitConnections, effectiveIkeVersion); + List existingConnections = _vpnConnectionDao.listByCustomerGatewayId(id); + if (existingConnections != null) { + for (Site2SiteVpnConnectionVO connection : existingConnections) { + Site2SiteVpnGatewayVO gateway = _vpnGatewayDao.findById(connection.getVpnGatewayId()); + if (gateway == null) { + throw new CloudRuntimeException(String.format( + "Unable to validate customer gateway %s because VPN gateway %s does not exist", + id, connection.getVpnGatewayId())); + } + validateCustomerGatewayForVpnGateway(proposedGateway, gateway); + } + } + gw.setName(name); gw.setGatewayIp(gatewayIp); gw.setGuestCidrList(guestCidrList); @@ -760,117 +922,125 @@ public Site2SiteCustomerGateway updateCustomerGateway(UpdateVpnCustomerGatewayCm private void setupVpnConnection(Account caller, Long vpnCustomerGwIp) { List conns = _vpnConnectionDao.listByCustomerGatewayId(vpnCustomerGwIp); if (conns != null) { - for (Site2SiteVpnConnection conn : conns) { - try { - _accountMgr.checkAccess(caller, null, false, conn); - } catch (PermissionDeniedException e) { - // Just don't restart this connection, as the user has no rights to it - // Maybe should issue a notification to the system? - logger.info("Site2SiteVpnManager:updateCustomerGateway() Not resetting VPN connection {} as user lacks permission", conn); - continue; - } - - if (conn.getState() == State.Pending) { - // Vpn connection cannot be reset when the state is Pending - continue; + for (Site2SiteVpnConnectionVO listedConnection : conns) { + Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(listedConnection.getId()); + if (conn == null) { + throw new CloudRuntimeException("Unable to acquire lock for restarting VPN connection with ID " + listedConnection.getId()); } try { + try { + _accountMgr.checkAccess(caller, null, false, conn); + } catch (PermissionDeniedException e) { + logger.info("Site2SiteVpnManager:updateCustomerGateway() Not resetting VPN connection {} as user lacks permission", conn); + continue; + } + if (conn.getState() == State.Pending || conn.getState() == State.Removed) { + continue; + } if (conn.getState() == State.Connected || conn.getState() == State.Connecting || conn.getState() == State.Error) { - stopVpnConnection(conn.getId()); + stopVpnConnectionLocked(conn, false); } - startVpnConnection(conn.getId()); + startVpnConnectionLocked(conn); } catch (ResourceUnavailableException e) { - // Should never get here, as we are looping on the actual connections, but we must handle it regardless - logger.warn("Failed to update VPN connection"); + logger.warn("Failed to restart VPN connection {} after updating its customer gateway: {}", + conn.getUuid(), e.getMessage()); + } finally { + _vpnConnectionDao.releaseFromLockTable(conn.getId()); } } } } @Override + @DB @ActionEvent(eventType = EventTypes.EVENT_S2S_VPN_CONNECTION_DELETE, eventDescription = "deleting s2s vpn connection", create = true) public boolean deleteVpnConnection(DeleteVpnConnectionCmd cmd) throws ResourceUnavailableException { Account caller = CallContext.current().getCallingAccount(); Long id = cmd.getId(); - Site2SiteVpnConnectionVO conn = _vpnConnectionDao.findById(id); + Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id); if (conn == null) { - throw new InvalidParameterValueException("Fail to find site to site VPN connection " + id + " to delete!"); + if (_vpnConnectionDao.findById(id) == null) { + throw new InvalidParameterValueException("Fail to find site to site VPN connection " + id + " to delete!"); + } + throw new CloudRuntimeException("Unable to acquire lock for deleting VPN connection with ID " + id); } - CallContext.current().setEventDetails(" ID: " + conn.getUuid()); - - _accountMgr.checkAccess(caller, null, false, conn); + try { + CallContext.current().setEventDetails(" ID: " + conn.getUuid()); + _accountMgr.checkAccess(caller, null, false, conn); - if (conn.getState() != State.Pending) { - stopVpnConnection(id); - } + stopVpnConnectionLocked(conn, true); - conn.setState(State.Removed); - _vpnConnectionDao.update(id, conn); + conn.setState(State.Removed); + _vpnConnectionDao.update(id, conn); - final Site2SiteVpnGateway vpnGateway = _vpnGatewayDao.findById(conn.getVpnGatewayId()); - try { - vpcManager.applyStaticRouteForVpcVpnIfNeeded(vpnGateway.getVpcId(), false); - } catch (ResourceUnavailableException | CloudRuntimeException e) { - logger.error("Unable to apply static routes for vpc " + vpnGateway.getVpcId() + "as part of deletion of VPN connection, due to " + e.getMessage()); - } + Site2SiteVpnGateway vpnGateway = getVpnGatewayForConnection(conn); + try { + vpcManager.applyStaticRouteForVpcVpnIfNeeded(vpnGateway.getVpcId(), false); + } catch (ResourceUnavailableException | CloudRuntimeException e) { + logger.error("Unable to apply static routes for vpc {} as part of deletion of VPN connection, due to {}", + vpnGateway.getVpcId(), e.getMessage()); + } - _vpnConnectionDao.remove(id); + _vpnConnectionDao.remove(id); - return true; + return true; + } finally { + _vpnConnectionDao.releaseFromLockTable(conn.getId()); + } } - @DB - private void stopVpnConnection(Long id) throws ResourceUnavailableException { - Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id); - if (conn == null) { - throw new CloudRuntimeException("Unable to acquire lock for stopping VPN connection with ID " + id); + private void stopVpnConnectionLocked(Site2SiteVpnConnectionVO conn, boolean deleting) throws ResourceUnavailableException { + if (conn.getState() == State.Pending && !deleting) { + throw new InvalidParameterValueException("Site to site VPN connection with specified id is currently Pending, unable to Disconnect!"); } - try { - if (conn.getState() == State.Pending) { - throw new InvalidParameterValueException("Site to site VPN connection with specified id is currently Pending, unable to Disconnect!"); - } - conn.setState(State.Disconnected); - _vpnConnectionDao.persist(conn); + conn.setState(State.Disconnected); + _vpnConnectionDao.persist(conn); - boolean result = true; - for (Site2SiteVpnServiceProvider element : _s2sProviders) { - result = result & element.stopSite2SiteVpn(conn); + Site2SiteVpnGateway vpnGateway = getVpnGatewayForConnection(conn); + try { + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForGateway(vpnGateway); + if (provider == null) { + throw new CloudRuntimeException(String.format( + "No Site-to-Site VPN provider owns gateway %s", vpnGateway.getId())); } - + boolean result = deleting ? provider.deleteSite2SiteVpn(conn) : provider.stopSite2SiteVpn(conn); if (!result) { - conn.setState(State.Error); - _vpnConnectionDao.persist(conn); - throw new ResourceUnavailableException("Failed to apply site-to-site VPN", Site2SiteVpnConnection.class, id); + throw new ResourceUnavailableException("Failed to apply site-to-site VPN", + Site2SiteVpnConnection.class, conn.getId()); } - } finally { - _vpnConnectionDao.releaseFromLockTable(conn.getId()); + } catch (ResourceUnavailableException | RuntimeException e) { + conn.setState(State.Error); + _vpnConnectionDao.persist(conn); + throw e; } } @Override + @DB @ActionEvent(eventType = EventTypes.EVENT_S2S_VPN_CONNECTION_RESET, eventDescription = "reseting s2s vpn connection", create = true) public Site2SiteVpnConnection resetVpnConnection(ResetVpnConnectionCmd cmd) throws ResourceUnavailableException { Account caller = CallContext.current().getCallingAccount(); Long id = cmd.getId(); - Site2SiteVpnConnectionVO conn = _vpnConnectionDao.findById(id); + Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id); if (conn == null) { - throw new InvalidParameterValueException("Fail to find site to site VPN connection " + id + " to reset!"); + if (_vpnConnectionDao.findById(id) == null) { + throw new InvalidParameterValueException("Fail to find site to site VPN connection " + id + " to reset!"); + } + throw new CloudRuntimeException("Unable to acquire lock for resetting VPN connection with ID " + id); } - CallContext.current().setEventDetails(" ID: " + conn.getUuid()); - _accountMgr.checkAccess(caller, null, false, conn); - - // Set vpn state to disconnected - conn.setState(State.Disconnected); - _vpnConnectionDao.persist(conn); + try { + CallContext.current().setEventDetails(" ID: " + conn.getUuid()); + _accountMgr.checkAccess(caller, null, false, conn); - // Stop and start the connection again - stopVpnConnection(id); - startVpnConnection(id); - conn = _vpnConnectionDao.findById(id); - return conn; + conn.setState(State.Disconnected); + stopVpnConnectionLocked(conn, false); + return startVpnConnectionLocked(conn); + } finally { + _vpnConnectionDao.releaseFromLockTable(conn.getId()); + } } @Override @@ -1067,6 +1237,15 @@ public List getConnectionsForRouter(DomainRouterVO rou if (router.getVpcId() == null) { return conns; } + Site2SiteVpnGatewayVO gateway = _vpnGatewayDao.findByVpcId(vpcId); + if (gateway == null) { + return conns; + } + Site2SiteVpnServiceProvider provider = getVpnServiceProviderForGateway(gateway); + if (!(provider instanceof NetworkElement) + || !Network.Provider.VPCVirtualRouter.equals(((NetworkElement) provider).getProvider())) { + return conns; + } conns.addAll(_vpnConnectionDao.listByVpcId(vpcId)); return conns; } diff --git a/server/src/test/java/com/cloud/api/ApiServletTest.java b/server/src/test/java/com/cloud/api/ApiServletTest.java index c5ee9f58154d..8351223b6ec8 100644 --- a/server/src/test/java/com/cloud/api/ApiServletTest.java +++ b/server/src/test/java/com/cloud/api/ApiServletTest.java @@ -552,4 +552,62 @@ public void isStateChangingCommandNotUsingPOSTReturnsFalseForUpdateConfiguration boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); Assert.assertTrue(result); } + + @Test + public void shouldNotLogRequestParametersForAddObjectStoragePool() { + boolean result = servlet.shouldLogRequestParameters("addObjectStoragePool", new HashMap<>()); + + Assert.assertFalse(result); + } + + @Test + public void shouldLogRequestParametersForCommandWithoutSensitiveParameters() { + boolean result = servlet.shouldLogRequestParameters("listZones", new HashMap<>()); + + Assert.assertTrue(result); + } + + @Test + public void shouldNotLogRequestParametersContainingUserData() { + Map params = new HashMap<>(); + params.put(ApiConstants.USER_DATA, new String[] {"sensitive-user-data"}); + + boolean result = servlet.shouldLogRequestParameters("deployVirtualMachine", params); + + Assert.assertFalse(result); + } + + @Test + public void shouldReplaceQueryStringContainingUserDataWithCommandName() { + Map params = new HashMap<>(); + params.put(ApiConstants.USER_DATA, new String[] {"SYNTHETIC_USER_DATA"}); + String queryString = "command=deployVirtualMachine&userdata=SYNTHETIC_USER_DATA"; + + String result = servlet.getCleanQueryString("deployVirtualMachine", queryString, params); + + Assert.assertEquals("command=deployVirtualMachine", result); + Assert.assertFalse(result.contains("SYNTHETIC_USER_DATA")); + } + + @Test + public void shouldReplaceSensitiveQueryStringWithCommandName() { + Map params = new HashMap<>(); + String queryString = "command=addObjectStoragePool&details%5B1%5D.value=SYNTHETIC_SECRET_KEY"; + + String result = servlet.getCleanQueryString("addObjectStoragePool", queryString, params); + + Assert.assertEquals("command=addObjectStoragePool", result); + Assert.assertFalse(result.contains("SYNTHETIC_SECRET_KEY")); + } + + @Test + public void shouldKeepOrdinaryQueryString() { + Map params = new HashMap<>(); + String queryString = "command=listZones&response=json"; + + String result = servlet.getCleanQueryString("listZones", queryString, params); + + Assert.assertEquals(queryString, result); + } + } diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 9a0b150780e4..86a4041fc230 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -18,11 +18,14 @@ import com.cloud.alert.AlertManager; import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.AccountVlanMapVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.VlanVO; +import com.cloud.dc.dao.AccountVlanMapDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DataCenterIpAddressDao; import com.cloud.dc.dao.DedicatedResourceDao; +import com.cloud.dc.dao.DomainVlanMapDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.domain.Domain; @@ -34,10 +37,15 @@ import com.cloud.network.NetworkService; import com.cloud.network.Networks; import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetrisProviderDao; import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.dao.NsxVrfGatewayPlacementDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.element.NsxProviderVO; +import com.cloud.network.element.NsxVrfGatewayVO; +import com.cloud.network.nsx.NsxService; import com.cloud.offering.DiskOffering; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.NetworkOfferingVO; @@ -49,10 +57,12 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManagerImpl; +import com.cloud.user.ResourceLimitService; import com.cloud.user.User; import com.cloud.utils.DomainHelper; import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @@ -147,6 +157,10 @@ public class ConfigurationManagerImplTest { @Mock NsxProviderDao nsxProviderDao; @Mock + NsxVrfGatewayDao nsxVrfGatewayDao; + @Mock + NsxVrfGatewayPlacementDao nsxVrfGatewayPlacementDao; + @Mock NetrisProviderDao netrisProviderDao; @Mock DataCenterDao zoneDao; @@ -169,6 +183,10 @@ public class ConfigurationManagerImplTest { @Mock VlanDao vlanDao; @Mock + AccountVlanMapDao accountVlanMapDao; + @Mock + DomainVlanMapDao domainVlanMapDao; + @Mock VMTemplateZoneDao vmTemplateZoneDao; @Mock CapacityDao capacityDao; @@ -190,6 +208,8 @@ public class ConfigurationManagerImplTest { StoragePoolDetailsDao storagePoolDetailsDao; @Mock DomainHelper domainHelper; + @Mock + ResourceLimitService resourceLimitService; DeleteZoneCmd deleteZoneCmd; CreateNetworkOfferingCmd createNetworkOfferingCmd; @@ -1410,4 +1430,208 @@ public void testGetExternalNetworkProviderReturnsNullWhenNoExternalProviders() { mapWithEmptySet.put(Network.Service.Firewall, Collections.emptySet()); Assert.assertNull(ConfigurationManagerImpl.getExternalNetworkProvider(null, mapWithEmptySet)); } + + @Test + public void testValidateNetworkOfferingDetailsAcceptsNsxSegmentProfilesForNsxOffering() { + Map details = Map.of( + NetworkOffering.Detail.NsxIpDiscoveryProfileId, "cloudstack-ip-discovery", + NetworkOffering.Detail.NsxMacDiscoveryProfileId, "cloudstack-mac-discovery", + NetworkOffering.Detail.NsxSegmentSecurityProfileId, "cloudstack-segment-security"); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.Nsx)); + + configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap); + } + + @Test + public void testValidateNetworkOfferingDetailsRejectsWhitespaceInNsxProfileId() { + Map details = Map.of( + NetworkOffering.Detail.NsxMacDiscoveryProfileId, " mac-profile "); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.Nsx)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap)); + } + + @Test + public void testValidateNetworkOfferingDetailsRejectsNsxSegmentProfileForNonNsxOffering() { + Map details = Map.of( + NetworkOffering.Detail.NsxIpDiscoveryProfileId, "cloudstack-ip-discovery"); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.VPCVirtualRouter)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap)); + } + + @Test + public void testValidateNetworkOfferingDetailsRejectsNsxSegmentProfileWhenOnlyAnotherServiceUsesNsx() { + Map details = Map.of( + NetworkOffering.Detail.NsxIpDiscoveryProfileId, "cloudstack-ip-discovery"); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.VPCVirtualRouter), + Network.Service.Lb, Set.of(Network.Provider.Nsx)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap)); + } + + @Test + public void testUpdateVlanRejectsSubnetMetadataChangeForRegisteredVrfRange() throws Exception { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(1L); + when(vlan.getDataCenterId()).thenReturn(2L); + when(vlan.getVlanGateway()).thenReturn("192.0.2.1"); + when(vlanDao.findById(1L)).thenReturn(vlan); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getNsxTier0Name()).thenReturn("CS-VRF-001"); + when(nsxVrfGatewayDao.findByPublicVlan(1L)).thenReturn(gateway); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); + + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(NsxService.getVrfZoneLockName(2L))).thenReturn(lock); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.updateVlanAndPublicIpRange(1L, null, null, + "192.0.2.254", null, null, null, null, null, null)); + } + + verify(lock).unlock(); + verify(lock).releaseRef(); + } + + @Test + public void testUpdateVlanRejectsSystemVmFlagChangeForRegisteredVrfRange() throws Exception { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(1L); + when(vlan.getDataCenterId()).thenReturn(2L); + when(vlan.getVlanGateway()).thenReturn("192.0.2.1"); + when(vlanDao.findById(1L)).thenReturn(vlan); + IPAddressVO address = mock(IPAddressVO.class); + when(address.isForSystemVms()).thenReturn(false); + when(publicIpAddressDao.listByVlanId(1L)).thenReturn(List.of(address)); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getNsxTier0Name()).thenReturn("CS-VRF-001"); + when(nsxVrfGatewayDao.findByPublicVlan(1L)).thenReturn(gateway); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); + + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(NsxService.getVrfZoneLockName(2L))).thenReturn(lock); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.updateVlanAndPublicIpRange(1L, null, null, + null, null, null, null, null, null, true)); + } + + verify(lock).unlock(); + verify(lock).releaseRef(); + } + + @Test + public void testDeleteVlanRejectsRegisteredVrfRange() { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(1L); + when(vlan.getDataCenterId()).thenReturn(2L); + when(vlanDao.findById(1L)).thenReturn(vlan); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getNsxTier0Name()).thenReturn("CS-VRF-001"); + when(nsxVrfGatewayDao.findByPublicVlan(1L)).thenReturn(gateway); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); + + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(NsxService.getVrfZoneLockName(2L))).thenReturn(lock); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.deleteVlanAndPublicIpRange(1L, 1L, accountMock)); + } + + verify(lock).unlock(); + verify(lock).releaseRef(); + } + + @Test + public void testOwnerRangeCleanupReleasesVrfAssignmentWhenNoPlacementsRemain() { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(1L); + when(vlan.getDataCenterId()).thenReturn(2L); + when(vlanDao.findById(1L)).thenReturn(vlan); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getId()).thenReturn(3L); + when(gateway.getAccountId()).thenReturn(42L); + when(gateway.getNsxTier0Name()).thenReturn("CS-VRF-001"); + when(nsxVrfGatewayDao.findByPublicVlan(1L)).thenReturn(gateway); + when(nsxVrfGatewayDao.update(3L, gateway)).thenReturn(true); + AccountVlanMapVO accountVlanMap = mock(AccountVlanMapVO.class); + when(accountVlanMap.getId()).thenReturn(4L); + when(accountVlanMap.getAccountId()).thenReturn(42L); + when(accountVlanMapDao.listAccountVlanMapsByVlan(1L)).thenReturn(List.of(accountVlanMap)); + when(accountVlanMapDao.remove(4L)).thenReturn(true); + when(publicIpAddressDao.listByVlanId(1L)).thenReturn(List.of()); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); + + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(NsxService.getVrfZoneLockName(2L))).thenReturn(lock); + + Boolean released = ReflectionTestUtils.invokeMethod(configurationManagerImplSpy, + "releaseOwnerPublicIpRange", 1L, userMock, accountMock, 42L, null); + Assert.assertTrue(released); + } + + verify(gateway).setScope(null); + verify(gateway).setAccountId(null); + verify(gateway).setDomainId(null); + verify(nsxVrfGatewayDao).update(3L, gateway); + } + + @Test + public void testOwnerRangeCleanupRejectsVrfAssignmentWithPlacements() { + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(1L); + when(vlan.getDataCenterId()).thenReturn(2L); + when(vlanDao.findById(1L)).thenReturn(vlan); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getId()).thenReturn(3L); + when(gateway.getAccountId()).thenReturn(42L); + when(gateway.getNsxTier0Name()).thenReturn("CS-VRF-001"); + when(nsxVrfGatewayDao.findByPublicVlan(1L)).thenReturn(gateway); + when(nsxVrfGatewayPlacementDao.countByGatewayId(3L)).thenReturn(1L); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); + + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(NsxService.getVrfZoneLockName(2L))).thenReturn(lock); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> ReflectionTestUtils.invokeMethod(configurationManagerImplSpy, + "releaseOwnerPublicIpRange", 1L, userMock, accountMock, 42L, null)); + } + + verify(nsxVrfGatewayDao, Mockito.never()).update(anyLong(), any()); + } + + @Test + public void testValidateNetworkOfferingDetailsRejectsBlankNsxSegmentProfile() { + Map details = Map.of(NetworkOffering.Detail.NsxMacDiscoveryProfileId, " "); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.Nsx)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap)); + } + + @Test + public void testValidateNetworkOfferingDetailsRejectsOversizedNsxSegmentProfile() { + Map details = Map.of( + NetworkOffering.Detail.NsxSegmentSecurityProfileId, "x".repeat(256)); + Map> serviceProviderMap = Map.of( + Network.Service.Connectivity, Set.of(Network.Provider.Nsx)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> configurationManagerImplSpy.validateNtwkOffDetails(details, serviceProviderMap)); + } } diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java index 8873d58de605..9012cb21a38c 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java @@ -57,6 +57,7 @@ import com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.NsxVrfGatewayDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.offering.DiskOffering; @@ -76,6 +77,7 @@ import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; @@ -110,6 +112,7 @@ import org.junit.Test; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; @@ -200,6 +203,8 @@ public class ConfigurationManagerTest { @Mock NetworkDao _networkDao; @Mock + NsxVrfGatewayDao nsxVrfGatewayDao; + @Mock PhysicalNetworkDao _physicalNetworkDao; @Mock ImageStoreDao _imageStoreDao; @@ -316,25 +321,32 @@ public void testReleasePublicIpRange() throws Exception { logger.info("Running tests for DedicatePublicIpRange API"); - /* - * TEST 1: given valid parameters and no allocated public ip's in the range ReleasePublicIpRange should succeed - */ - runReleasePublicIpRangePostiveTest1(); + GlobalLock lock = mock(GlobalLock.class); + when(lock.lock(30)).thenReturn(true); - /* - * TEST 2: given valid parameters ReleasePublicIpRange should succeed - */ - runReleasePublicIpRangePostiveTest2(); + try (MockedStatic globalLock = Mockito.mockStatic(GlobalLock.class)) { + globalLock.when(() -> GlobalLock.getInternLock(anyString())).thenReturn(lock); - /* - * TEST 3: given range doesn't exist - */ - runReleasePublicIpRangeInvalidIpRange(); + /* + * TEST 1: given valid parameters and no allocated public ip's in the range ReleasePublicIpRange should succeed + */ + runReleasePublicIpRangePostiveTest1(); - /* - * TEST 4: given range is not dedicated to any account - */ - runReleaseNonDedicatedPublicIpRange(); + /* + * TEST 2: given valid parameters ReleasePublicIpRange should succeed + */ + runReleasePublicIpRangePostiveTest2(); + + /* + * TEST 3: given range doesn't exist + */ + runReleasePublicIpRangeInvalidIpRange(); + + /* + * TEST 4: given range is not dedicated to any account + */ + runReleaseNonDedicatedPublicIpRange(); + } } void runDedicatePublicIpRangePostiveTest() throws Exception { diff --git a/server/src/test/java/com/cloud/network/IpAddressManagerTest.java b/server/src/test/java/com/cloud/network/IpAddressManagerTest.java index d12419b66e0a..0188db9f9502 100644 --- a/server/src/test/java/com/cloud/network/IpAddressManagerTest.java +++ b/server/src/test/java/com/cloud/network/IpAddressManagerTest.java @@ -57,6 +57,10 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.NetrisProviderDao; +import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.element.NsxVrfGatewayVO; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.StaticNat; import com.cloud.network.rules.StaticNatImpl; @@ -67,6 +71,7 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; import com.cloud.utils.net.Ip; @RunWith(MockitoJUnitRunner.class) @@ -122,6 +127,18 @@ public class IpAddressManagerTest { @Mock VlanDao vlanDao; + @Mock + AccountDao accountDao; + + @Mock + NsxProviderDao nsxProviderDao; + + @Mock + NsxVrfGatewayDao nsxVrfGatewayDao; + + @Mock + NetrisProviderDao netrisProviderDao; + final long dummyID = 1L; final String UUID = "uuid"; @@ -147,6 +164,110 @@ public void setup() throws ResourceUnavailableException { Mockito.when(networkOfferingDao.findById(Mockito.anyLong())).thenReturn(networkOfferingVO); } + @Test + public void testAllocateIpRejectsReservedAddressOutsideRequiredVlan() throws Exception { + com.cloud.dc.DataCenter zone = mock(com.cloud.dc.DataCenter.class); + when(zone.getId()).thenReturn(2L); + when(zone.getAllocationState()).thenReturn(com.cloud.org.Grouping.AllocationState.Enabled); + when(accountDao.acquireInLockTable(account.getId())).thenReturn(account); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getZoneId()).thenReturn(2L); + when(nsxVrfGatewayDao.findByPublicVlan(10L)).thenReturn(gateway); + IPAddressVO reserved = mock(IPAddressVO.class); + when(reserved.getVlanId()).thenReturn(9L); + when(ipAddressDao.findByAccountIdAndZoneIdAndStateAndIpAddress(account.getId(), 2L, + IpAddress.State.Reserved, "192.0.2.10")).thenReturn(reserved); + + Assert.assertThrows(com.cloud.exception.InvalidParameterValueException.class, + () -> ipAddressManager.allocateIpFromNsxVrfPublicRange(account, false, account, null, zone, + true, "192.0.2.10", 10L)); + + verify(accountDao).releaseFromLockTable(account.getId()); + } + + @Test + public void testAllocateIpWithoutRequiredVlanKeepsReservedAddressPath() throws Exception { + com.cloud.dc.DataCenter zone = mock(com.cloud.dc.DataCenter.class); + when(zone.getId()).thenReturn(2L); + when(zone.getAllocationState()).thenReturn(com.cloud.org.Grouping.AllocationState.Enabled); + when(accountDao.acquireInLockTable(account.getId())).thenReturn(account); + IPAddressVO reserved = mock(IPAddressVO.class); + when(reserved.getId()).thenReturn(5L); + when(reserved.getVlanId()).thenReturn(9L); + when(ipAddressDao.findByAccountIdAndZoneIdAndStateAndIpAddress(account.getId(), 2L, + IpAddress.State.Reserved, "192.0.2.10")).thenReturn(reserved); + com.cloud.dc.VlanVO vlan = mock(com.cloud.dc.VlanVO.class); + when(vlanDao.findById(9L)).thenReturn(vlan); + + IpAddress result = ipAddressManager.allocateIp(account, false, account, null, zone, true, + "192.0.2.10"); + + Assert.assertEquals(5L, result.getId()); + verify(accountDao).releaseFromLockTable(account.getId()); + } + + @Test + public void testGenericAllocateIpRejectsReservedAddressFromRegisteredNsxVrfRange() throws Exception { + com.cloud.dc.DataCenter zone = mock(com.cloud.dc.DataCenter.class); + when(zone.getId()).thenReturn(2L); + when(zone.getAllocationState()).thenReturn(com.cloud.org.Grouping.AllocationState.Enabled); + when(accountDao.acquireInLockTable(account.getId())).thenReturn(account); + IPAddressVO reserved = mock(IPAddressVO.class); + when(reserved.getVlanId()).thenReturn(9L); + when(ipAddressDao.findByAccountIdAndZoneIdAndStateAndIpAddress(account.getId(), 2L, + IpAddress.State.Reserved, "192.0.2.10")).thenReturn(reserved); + when(nsxVrfGatewayDao.findByPublicVlan(9L)).thenReturn(mock(NsxVrfGatewayVO.class)); + + Assert.assertThrows(com.cloud.exception.InvalidParameterValueException.class, + () -> ipAddressManager.allocateIp(account, false, account, null, zone, true, + "192.0.2.10")); + + verify(accountDao).releaseFromLockTable(account.getId()); + } + + @Test + public void testNsxVrfAllocateIpAcceptsReservedAddressFromItsRegisteredRange() throws Exception { + com.cloud.dc.DataCenter zone = mock(com.cloud.dc.DataCenter.class); + when(zone.getId()).thenReturn(2L); + when(zone.getAllocationState()).thenReturn(com.cloud.org.Grouping.AllocationState.Enabled); + when(accountDao.acquireInLockTable(account.getId())).thenReturn(account); + NsxVrfGatewayVO gateway = mock(NsxVrfGatewayVO.class); + when(gateway.getZoneId()).thenReturn(2L); + when(nsxVrfGatewayDao.findByPublicVlan(9L)).thenReturn(gateway); + IPAddressVO reserved = mock(IPAddressVO.class); + when(reserved.getId()).thenReturn(5L); + when(reserved.getVlanId()).thenReturn(9L); + when(ipAddressDao.findByAccountIdAndZoneIdAndStateAndIpAddress(account.getId(), 2L, + IpAddress.State.Reserved, "192.0.2.10")).thenReturn(reserved); + com.cloud.dc.VlanVO vlan = mock(com.cloud.dc.VlanVO.class); + when(vlanDao.findById(9L)).thenReturn(vlan); + + IpAddress result = ipAddressManager.allocateIpFromNsxVrfPublicRange(account, false, account, null, + zone, true, "192.0.2.10", 9L); + + Assert.assertEquals(5L, result.getId()); + verify(accountDao).releaseFromLockTable(account.getId()); + } + + @Test + public void testGenericAllocationRejectsRangeRegisteredAfterAddressLock() { + IPAddressVO address = mock(IPAddressVO.class); + when(address.getVlanId()).thenReturn(9L); + when(nsxVrfGatewayDao.lockByPublicVlan(9L)).thenReturn(mock(NsxVrfGatewayVO.class)); + + Assert.assertThrows(com.cloud.exception.InsufficientAddressCapacityException.class, + () -> ipAddressManager.validateSelectedPublicIpRangeRegistration(address, false, 2L)); + } + + @Test + public void testNsxVrfAllocationAllowsItsRegisteredRangeAfterAddressLock() throws Exception { + IPAddressVO address = mock(IPAddressVO.class); + + ipAddressManager.validateSelectedPublicIpRangeRegistration(address, true, 2L); + + verify(nsxVrfGatewayDao, never()).lockByPublicVlan(9L); + } + @Test public void testGetStaticNatSourceIps() { String publicIpAddress = "192.168.1.3"; diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index cd7d40d68951..fa63fbdf87b0 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import org.apache.cloudstack.alert.AlertService; @@ -72,7 +73,10 @@ import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.VlanDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; @@ -85,6 +89,8 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.dao.NsxVrfGatewayDao; +import com.cloud.network.element.NsxVrfGatewayVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.dao.PublicIpQuarantineDao; @@ -167,6 +173,8 @@ public class NetworkServiceImplTest { @Mock IPAddressDao ipAddressDao; @Mock + VlanDao vlanDao; + @Mock ConfigurationManager configMgr; @Mock ConfigKey publicMtuKey; @@ -237,6 +245,8 @@ public class NetworkServiceImplTest { private Ip ipMock; @Mock private NsxProviderDao nsxProviderDao; + @Mock + private NsxVrfGatewayDao nsxVrfGatewayDao; private static Date beforeDate; @@ -274,6 +284,110 @@ public class NetworkServiceImplTest { private Long zoneId = 10L; private Long networkId = 11L; + @Test + public void testAllocateIpWithoutNsxPlacementKeepsStandardAllocationPath() throws Exception { + long allocationZoneId = 2L; + IpAddress allocatedIp = mock(IpAddress.class); + when(entityMgr.findById(DataCenter.class, allocationZoneId)).thenReturn(dc); + when(ipAddressManagerMock.allocateIp(accountMock, false, accountMock, null, dc, true, null)) + .thenReturn(allocatedIp); + + try (MockedStatic componentContext = Mockito.mockStatic(ComponentContext.class)) { + IpAddress result = service.allocateIP(accountMock, allocationZoneId, null, null, true, null); + + Assert.assertEquals(allocatedIp, result); + componentContext.verifyNoInteractions(); + } + + Mockito.verify(ipAddressManagerMock).allocateIp(accountMock, false, accountMock, null, dc, true, null); + } + + @Test + public void testAllocateIpForNsxVpcReservesPlacementAfterAccessCheck() throws Exception { + long allocationZoneId = 2L; + IpAddress allocatedIp = mock(IpAddress.class); + when(accountMock.getId()).thenReturn(42L); + when(entityMgr.findById(DataCenter.class, allocationZoneId)).thenReturn(dc); + VpcVO nsxVpc = mock(VpcVO.class); + when(nsxVpc.getId()).thenReturn(9L); + when(nsxVpc.getZoneId()).thenReturn(allocationZoneId); + when(nsxVpc.getAccountId()).thenReturn(42L); + when(nsxVpc.getDomainId()).thenReturn(6L); + when(nsxVpc.getVpcOfferingId()).thenReturn(7L); + when(vpcDao.findById(9L)).thenReturn(nsxVpc); + when(vpcMgr.getVpcOffSvcProvidersMap(7L)).thenReturn( + Map.of(Network.Service.SourceNat, Set.of(Network.Provider.Nsx))); + NsxService nsxService = mock(NsxService.class); + when(nsxService.reserveTier1PlacementAndGetPublicVlanId(allocationZoneId, 42L, 6L, 9L, null)) + .thenReturn(73L); + when(ipAddressManagerMock.allocateIpFromNsxVrfPublicRange(accountMock, false, accountMock, null, dc, + true, null, 73L)) + .thenReturn(allocatedIp); + + try (MockedStatic componentContext = Mockito.mockStatic(ComponentContext.class)) { + componentContext.when(() -> ComponentContext.getDelegateComponentOfType(NsxService.class)) + .thenReturn(nsxService); + + Assert.assertEquals(allocatedIp, + service.allocateIP(accountMock, allocationZoneId, null, 9L, true, null)); + } + + org.mockito.InOrder order = Mockito.inOrder(accountManager, nsxService, ipAddressManagerMock); + order.verify(accountManager).checkAccess(accountMock, null, false, accountMock); + order.verify(nsxService).reserveTier1PlacementAndGetPublicVlanId(allocationZoneId, 42L, 6L, 9L, null); + order.verify(ipAddressManagerMock).allocateIpFromNsxVrfPublicRange(accountMock, false, accountMock, + null, dc, true, null, 73L); + } + + @Test + public void testAllocateIpRejectsForeignVpcBeforePlacementReservation() throws Exception { + long allocationZoneId = 2L; + when(accountMock.getId()).thenReturn(42L); + when(entityMgr.findById(DataCenter.class, allocationZoneId)).thenReturn(dc); + VpcVO foreignVpc = mock(VpcVO.class); + when(foreignVpc.getZoneId()).thenReturn(allocationZoneId); + when(foreignVpc.getAccountId()).thenReturn(43L); + when(vpcDao.findById(9L)).thenReturn(foreignVpc); + + try (MockedStatic componentContext = Mockito.mockStatic(ComponentContext.class)) { + Assert.assertThrows(InvalidParameterValueException.class, + () -> service.allocateIP(accountMock, allocationZoneId, null, 9L, true, null)); + componentContext.verifyNoInteractions(); + } + + Mockito.verifyNoInteractions(ipAddressManagerMock); + } + + @Test + public void testAllocatePortableIpRejectsNsxTier1Vpc() { + VpcVO nsxVpc = mock(VpcVO.class); + when(nsxVpc.getVpcOfferingId()).thenReturn(7L); + when(vpcDao.findById(9L)).thenReturn(nsxVpc); + when(vpcMgr.getVpcOffSvcProvidersMap(7L)).thenReturn( + Map.of(Network.Service.Gateway, Set.of(Network.Provider.Nsx))); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> service.allocatePortableIP(accountMock, 1, 2L, null, 9L)); + Mockito.verifyNoInteractions(ipAddressManagerMock); + } + + @Test + public void testReserveIpAddressRejectsRegisteredNsxVrfPublicRange() { + IPAddressVO ip = mock(IPAddressVO.class); + when(ip.getVlanId()).thenReturn(73L); + when(ipAddressDao.findById(5L)).thenReturn(ip); + VlanVO vlan = mock(VlanVO.class); + when(vlan.getId()).thenReturn(73L); + when(vlan.getVlanType()).thenReturn(Vlan.VlanType.VirtualNetwork); + when(vlanDao.findById(73L)).thenReturn(vlan); + when(nsxVrfGatewayDao.findByPublicVlan(73L)).thenReturn(mock(NsxVrfGatewayVO.class)); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> service.reserveIpAddress(accountMock, true, 5L)); + + Mockito.verify(ipAddressDao, Mockito.never()).persist(any(IPAddressVO.class)); + } + @BeforeClass public static void setUpBeforeClass() { Date date = new Date(); @@ -317,11 +431,13 @@ public void setup() throws Exception { service._networksDao = networkDao; service._nicDao = nicDao; service._ipAddressDao = ipAddressDao; + service._vlanDao = vlanDao; service.routerDao = routerDao; service.commandSetupHelper = commandSetupHelper; service.networkHelper = networkHelper; service._ipAddrMgr = ipAddressManagerMock; service.nsxProviderDao = nsxProviderDao; + service.nsxVrfGatewayDao = nsxVrfGatewayDao; callContextMocked = Mockito.mockStatic(CallContext.class); CallContext callContextMock = Mockito.mock(CallContext.class); callContextMocked.when(CallContext::current).thenReturn(callContextMock); @@ -670,6 +786,42 @@ public void testCheckAndUpdateNetworkNoUpdate() { Assert.assertFalse(service.checkAndUpdateNetworkDns(network1, Mockito.mock(NetworkOffering.class), ip4Dns[0], null, ip6Dns[0], null)); } + @Test + public void testMatchingNsxSegmentProfilesAllowOfferingUpgrade() { + long oldOfferingId = 1L; + long newOfferingId = 2L; + Map profiles = Map.of( + NetworkOffering.Detail.NsxIpDiscoveryProfileId, "ip-profile", + NetworkOffering.Detail.NsxMacDiscoveryProfileId, "mac-profile", + NetworkOffering.Detail.NsxSegmentSecurityProfileId, "security-profile"); + Mockito.when(networkModel.getNtwkOffDetails(oldOfferingId)).thenReturn(profiles); + Mockito.when(networkModel.getNtwkOffDetails(newOfferingId)).thenReturn(profiles); + + Assert.assertTrue(service.haveMatchingNsxSegmentProfiles(oldOfferingId, newOfferingId)); + } + + @Test + public void testDifferentNsxSegmentProfilesRejectOfferingUpgrade() { + long oldOfferingId = 1L; + long newOfferingId = 2L; + Mockito.when(networkModel.getNtwkOffDetails(oldOfferingId)).thenReturn(Map.of( + NetworkOffering.Detail.NsxMacDiscoveryProfileId, "old-mac-profile")); + Mockito.when(networkModel.getNtwkOffDetails(newOfferingId)).thenReturn(Map.of( + NetworkOffering.Detail.NsxMacDiscoveryProfileId, "new-mac-profile")); + + Assert.assertFalse(service.haveMatchingNsxSegmentProfiles(oldOfferingId, newOfferingId)); + } + + @Test + public void testAbsentNsxSegmentProfilesAllowOfferingUpgrade() { + long oldOfferingId = 1L; + long newOfferingId = 2L; + Mockito.when(networkModel.getNtwkOffDetails(oldOfferingId)).thenReturn(null); + Mockito.when(networkModel.getNtwkOffDetails(newOfferingId)).thenReturn(Map.of()); + + Assert.assertTrue(service.haveMatchingNsxSegmentProfiles(oldOfferingId, newOfferingId)); + } + @Test public void testCheckAndUpdateNetworkOfferingChangeReset() { NetworkVO networkVO = new NetworkVO(); @@ -1210,6 +1362,8 @@ public void checkAndSetSourceNatIp() { NetworkOffering ntwkOff = Mockito.mock(NetworkOffering.class); Long networkId = 7l; when(networkVO.getId()).thenReturn(networkId); + when(networkVO.getDataCenterId()).thenReturn(zoneId); + when(networkVO.getAccountId()).thenReturn(account.getId()); when(networkVO.getGuestType()).thenReturn(Network.GuestType.Isolated); when(networkDao.findById(networkId)).thenReturn(networkVO); when(entityMgr.findById(NetworkOffering.class, networkOfferingId)).thenReturn(ntwkOff); @@ -1218,7 +1372,8 @@ public void checkAndSetSourceNatIp() { when(networkVO.getId()).thenReturn(networkId); when(networkVO.getGuestType()).thenReturn(Network.GuestType.Isolated); try { - when(ipAddressManagerMock.allocateIp(any(), anyBoolean(), any(), any(), any(), any(), eq(srcNatIp))).thenReturn(ipAddress); + when(ipAddressManagerMock.allocateIp(any(), anyBoolean(), any(), any(), any(), any(), eq(srcNatIp))) + .thenReturn(ipAddress); service.checkAndSetRouterSourceNatIp(account, createNetworkCmd, networkVO); } catch (InsufficientAddressCapacityException | ResourceAllocationException e) { Assert.fail(e.getMessage()); diff --git a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java index 2acad0c2b45d..3f9b8d97a2a3 100644 --- a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java @@ -42,6 +42,7 @@ import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.network.element.NetworkElement; import com.cloud.network.element.VpcProvider; import com.cloud.network.router.CommandSetupHelper; @@ -76,6 +77,7 @@ import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.extension.ExtensionHelper; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.junit.After; @@ -171,6 +173,8 @@ public class VpcManagerImplTest { NetworkACLVO networkACLVOMock; @Mock RoutedIpv4Manager routedIpv4Manager; + @Mock + ConfigurationDao configDao; public static final long ACCOUNT_ID = 1; private AccountVO account; @@ -231,6 +235,7 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { manager._firewallDao = firewallDao; manager._networkAclDao = networkACLDaoMock; manager.routedIpv4Manager = routedIpv4Manager; + manager._configDao = configDao; CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); registerCallContext(); overrideDefaultConfigValue(NetworkService.AllowUsersToSpecifyVRMtu, "_defaultValue", "false"); @@ -400,6 +405,153 @@ public void testCreateVpcNetwork() throws InsufficientCapacityException, Resourc null, null, null, null, null, new Pair<>(1000, 1000), null); } + private NetworkVO prepareVpcGuestNetworkCreationMocks(Account accountMock, PhysicalNetwork physicalNetwork) + throws InsufficientCapacityException, ResourceAllocationException { + final long VPC_ID = 201L; + manager._maxNetworks = 3; + VpcVO vpcMockVO = Mockito.mock(VpcVO.class); + Vpc vpcMock = Mockito.mock(Vpc.class); + NetworkOffering offering = Mockito.mock(NetworkOffering.class); + List services = new ArrayList<>(); + services.add(Service.SourceNat); + + Mockito.lenient().when(vpcDao.getActiveVpcById(anyLong())).thenReturn(vpcMock); + Mockito.lenient().doNothing().when(accountManager).checkAccess(any(Account.class), nullable(SecurityChecker.AccessType.class), anyBoolean(), any(Vpc.class)); + Mockito.lenient().when(vpcMock.isRegionLevelVpc()).thenReturn(true); + Mockito.lenient().when(entityMgr.findById(NetworkOffering.class, 1L)).thenReturn(offering); + Mockito.lenient().when(vpcMock.getId()).thenReturn(VPC_ID); + Mockito.lenient().when(vpcDao.acquireInLockTable(VPC_ID)).thenReturn(vpcMockVO); + Mockito.lenient().when(networkDao.countVpcNetworks(anyLong())).thenReturn(1L); + Mockito.lenient().when(offering.getGuestType()).thenReturn(Network.GuestType.Isolated); + Mockito.lenient().when(networkModel.listNetworkOfferingServices(anyLong())).thenReturn(services); + Mockito.lenient().when(networkOfferingServiceMapDao.listByNetworkOfferingId(anyLong())).thenReturn(new ArrayList<>()); + Mockito.lenient().when(vpcMock.getCidr()).thenReturn("10.0.0.0/8"); + Mockito.lenient().when(vpcMock.getNetworkDomain()).thenReturn("cs1cloud.internal"); + + NetworkVO createdNetwork = Mockito.mock(NetworkVO.class); + Mockito.when(networkMgr.createGuestNetwork(1L, "vpcNet1", "vpc tier 1", null, + "10.10.10.0/24", null, false, "cs1cloud.internal", accountMock, null, + physicalNetwork, zoneId, null, null, 1L, null, null, + true, null, null, null, null, + null, null, null, null, null, new Pair<>(1000, 1000), null)).thenReturn(createdNetwork); + return createdNetwork; + } + + @Test + public void testCreateVpcNetworkDefaultsToAllowAclWhenOfferingSupportsNetworkAcl() throws InsufficientCapacityException, ResourceAllocationException { + Account accountMock = Mockito.mock(Account.class); + PhysicalNetwork physicalNetwork = Mockito.mock(PhysicalNetwork.class); + NetworkVO createdNetwork = prepareVpcGuestNetworkCreationMocks(accountMock, physicalNetwork); + Mockito.when(networkModel.areServicesSupportedByNetworkOffering(anyLong(), Mockito.eq(Service.NetworkACL))).thenReturn(true); + + manager.createVpcGuestNetwork(1L, "vpcNet1", "vpc tier 1", null, + "10.10.10.0/24", null, null, accountMock, null, physicalNetwork, + 1L, null, null, 1L, null, accountMock, + true, null, null, null, null, null, null, null, new Pair<>(1000, 1000), null); + + Mockito.verify(createdNetwork, times(1)).setNetworkACLId(NetworkACL.DEFAULT_ALLOW); + Mockito.verify(networkDao, times(1)).update(anyLong(), any(NetworkVO.class)); + } + + @Test + public void testGetDefaultAclIdForNewTierHonoursDenySetting() { + NetworkOffering offering = Mockito.mock(NetworkOffering.class); + Mockito.when(offering.getName()).thenReturn("Regular VPC tier offering"); + Mockito.when(entityMgr.findById(NetworkOffering.class, 1L)).thenReturn(offering); + overrideConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL, "default_deny"); + try { + Assert.assertEquals(Long.valueOf(NetworkACL.DEFAULT_DENY), manager.getDefaultAclIdForNewTier(1L, zoneId)); + } finally { + resetConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL); + } + } + + @Test + public void testGetDefaultAclIdForNewTierAlwaysAllowsForKubernetesOfferings() { + NetworkOffering offering = Mockito.mock(NetworkOffering.class); + Mockito.when(offering.getName()).thenReturn("DefaultNSXVPCNetworkOfferingforKubernetesService"); + Mockito.when(entityMgr.findById(NetworkOffering.class, 1L)).thenReturn(offering); + overrideConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL, "default_deny"); + try { + Assert.assertEquals(Long.valueOf(NetworkACL.DEFAULT_ALLOW), manager.getDefaultAclIdForNewTier(1L, zoneId)); + } finally { + resetConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL); + } + } + + @Test + public void testGetDefaultAclIdForNewTierAllowsForConfiguredKubernetesOffering() { + NetworkOffering offering = Mockito.mock(NetworkOffering.class); + Mockito.when(offering.getName()).thenReturn("MyCustomK8sTierOffering"); + Mockito.when(entityMgr.findById(NetworkOffering.class, 1L)).thenReturn(offering); + Mockito.when(configDao.getValue(VpcManagerImpl.KUBERNETES_CLUSTER_NETWORK_OFFERING_CONFIG_KEY)).thenReturn("MyCustomK8sTierOffering"); + overrideConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL, "default_deny"); + try { + Assert.assertEquals(Long.valueOf(NetworkACL.DEFAULT_ALLOW), manager.getDefaultAclIdForNewTier(1L, zoneId)); + } finally { + resetConfigKeyValue(VpcManager.VpcTierDefaultNetworkACL); + } + } + + private void overrideConfigKeyValue(ConfigKey configKey, String value) { + try { + Field valueField = ConfigKey.class.getDeclaredField("_value"); + valueField.setAccessible(true); + valueField.set(configKey, value); + + Field dynamicField = ConfigKey.class.getDeclaredField("_isDynamic"); + dynamicField.setAccessible(true); + dynamicField.setBoolean(configKey, false); + } catch (IllegalAccessException | NoSuchFieldException e) { + throw new RuntimeException("Failed to set ConfigKey value", e); + } + } + + private void resetConfigKeyValue(ConfigKey configKey) { + try { + Field valueField = ConfigKey.class.getDeclaredField("_value"); + valueField.setAccessible(true); + valueField.set(configKey, null); + + Field dynamicField = ConfigKey.class.getDeclaredField("_isDynamic"); + dynamicField.setAccessible(true); + dynamicField.setBoolean(configKey, true); + } catch (IllegalAccessException | NoSuchFieldException e) { + throw new RuntimeException("Failed to reset ConfigKey value", e); + } + } + + @Test + public void testCreateVpcNetworkKeepsNullAclWhenOfferingLacksNetworkAclService() throws InsufficientCapacityException, ResourceAllocationException { + Account accountMock = Mockito.mock(Account.class); + PhysicalNetwork physicalNetwork = Mockito.mock(PhysicalNetwork.class); + NetworkVO createdNetwork = prepareVpcGuestNetworkCreationMocks(accountMock, physicalNetwork); + Mockito.when(networkModel.areServicesSupportedByNetworkOffering(anyLong(), Mockito.eq(Service.NetworkACL))).thenReturn(false); + + manager.createVpcGuestNetwork(1L, "vpcNet1", "vpc tier 1", null, + "10.10.10.0/24", null, null, accountMock, null, physicalNetwork, + 1L, null, null, 1L, null, accountMock, + true, null, null, null, null, null, null, null, new Pair<>(1000, 1000), null); + + Mockito.verify(createdNetwork, times(1)).setNetworkACLId((Long) null); + } + + @Test + public void testCreateVpcNetworkKeepsExplicitlyProvidedAcl() throws InsufficientCapacityException, ResourceAllocationException { + final Long explicitAclId = 5L; + Account accountMock = Mockito.mock(Account.class); + PhysicalNetwork physicalNetwork = Mockito.mock(PhysicalNetwork.class); + NetworkVO createdNetwork = prepareVpcGuestNetworkCreationMocks(accountMock, physicalNetwork); + Mockito.when(networkModel.areServicesSupportedByNetworkOffering(anyLong(), Mockito.eq(Service.NetworkACL))).thenReturn(true); + + manager.createVpcGuestNetwork(1L, "vpcNet1", "vpc tier 1", null, + "10.10.10.0/24", null, null, accountMock, null, physicalNetwork, + 1L, null, null, 1L, explicitAclId, accountMock, + true, null, null, null, null, null, null, null, new Pair<>(1000, 1000), null); + + Mockito.verify(createdNetwork, times(1)).setNetworkACLId(explicitAclId); + } + @Test public void testUpdateVpcNetwork() throws ResourceUnavailableException, InsufficientCapacityException { long vpcId = 1L; diff --git a/server/src/test/java/com/cloud/network/vpn/RemoteAccessVpnManagerImplTest.java b/server/src/test/java/com/cloud/network/vpn/RemoteAccessVpnManagerImplTest.java index f8b4362e76b5..1a39a29e9f53 100644 --- a/server/src/test/java/com/cloud/network/vpn/RemoteAccessVpnManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpn/RemoteAccessVpnManagerImplTest.java @@ -15,8 +15,21 @@ package com.cloud.network.vpn; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.RemoteAccessVpnDao; +import com.cloud.network.dao.RemoteAccessVpnVO; +import com.cloud.network.element.NetworkElement; +import com.cloud.network.element.RemoteAccessVPNServiceProvider; +import com.cloud.network.rules.FirewallManager; +import com.cloud.network.vpc.Vpc; +import com.cloud.network.vpc.VpcManager; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; +import org.apache.cloudstack.context.CallContext; import junit.framework.TestCase; import org.junit.Assert; import org.junit.Test; @@ -27,13 +40,137 @@ import javax.naming.ConfigurationException; import java.lang.reflect.InvocationTargetException; +import java.util.List; @RunWith(MockitoJUnitRunner.class) public class RemoteAccessVpnManagerImplTest extends TestCase { + private static final long NETWORK_ID = 11L; + private static final long VPC_ID = 12L; + Class expectedException = InvalidParameterValueException.class; Class cloudRuntimeException = CloudRuntimeException.class; + private RemoteAccessVPNServiceProvider mockProvider(Network.Provider networkProvider) { + RemoteAccessVPNServiceProvider provider = Mockito.mock(RemoteAccessVPNServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + Mockito.when(((NetworkElement) provider).getProvider()).thenReturn(networkProvider); + return provider; + } + + private RemoteAccessVpnManagerImpl managerWithProvider(RemoteAccessVPNServiceProvider provider) { + RemoteAccessVpnManagerImpl manager = new RemoteAccessVpnManagerImpl(); + manager._vpnServiceProviders = List.of(provider); + manager._networkMgr = Mockito.mock(com.cloud.network.NetworkModel.class); + manager.vpcManager = Mockito.mock(VpcManager.class); + return manager; + } + + @Test + public void validateVpcRemoteAccessVpnAcceptsMappedVpcVirtualRouterProvider() { + RemoteAccessVPNServiceProvider provider = mockProvider(Network.Provider.VPCVirtualRouter); + RemoteAccessVpnManagerImpl manager = managerWithProvider(provider); + Vpc vpc = Mockito.mock(Vpc.class); + IPAddressVO ipAddress = Mockito.mock(IPAddressVO.class); + Mockito.when(vpc.getId()).thenReturn(VPC_ID); + Mockito.when(manager.vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, + Network.Provider.VPCVirtualRouter)).thenReturn(true); + Mockito.when(manager.vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.SourceNat, + Network.Provider.VPCVirtualRouter)).thenReturn(true); + Mockito.when(ipAddress.isSourceNat()).thenReturn(true); + + manager.validateIpAddressForVpnServiceOnVpc(vpc, ipAddress); + } + + @Test + public void validateVpcRemoteAccessVpnRejectsProviderWithoutRemoteAccessImplementationBeforePersistence() { + RemoteAccessVPNServiceProvider provider = mockProvider(Network.Provider.VPCVirtualRouter); + RemoteAccessVpnManagerImpl manager = managerWithProvider(provider); + Vpc vpc = Mockito.mock(Vpc.class); + IPAddressVO ipAddress = Mockito.mock(IPAddressVO.class); + Mockito.when(vpc.getId()).thenReturn(VPC_ID); + Mockito.when(manager.vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, + Network.Provider.VPCVirtualRouter)).thenReturn(false); + InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.validateIpAddressForVpnServiceOnVpc(vpc, ipAddress)); + + Assert.assertTrue(exception.getMessage().contains("does not implement the Remote Access VPN service")); + } + + @Test + public void validateNetworkRemoteAccessVpnAcceptsMappedVirtualRouterProvider() { + RemoteAccessVPNServiceProvider provider = mockProvider(Network.Provider.VirtualRouter); + RemoteAccessVpnManagerImpl manager = managerWithProvider(provider); + Network network = Mockito.mock(Network.class); + IPAddressVO ipAddress = Mockito.mock(IPAddressVO.class); + Mockito.when(network.getId()).thenReturn(NETWORK_ID); + Mockito.when(manager._networkMgr.isProviderSupportServiceInNetwork(NETWORK_ID, Network.Service.Vpn, + Network.Provider.VirtualRouter)).thenReturn(true); + Mockito.when(manager._networkMgr.isProviderSupportServiceInNetwork(NETWORK_ID, Network.Service.SourceNat, + Network.Provider.VirtualRouter)).thenReturn(true); + Mockito.when(ipAddress.isSourceNat()).thenReturn(true); + + manager.validateIpAddressForVpnServiceOnNetwork(network, ipAddress); + } + + @Test + public void startRemoteAccessVpnRejectsLegacyRecordWithoutMappedProvider() throws ResourceUnavailableException { + RemoteAccessVPNServiceProvider provider = mockProvider(Network.Provider.VPCVirtualRouter); + RemoteAccessVpnManagerImpl manager = managerWithProvider(provider); + manager._remoteAccessVpnDao = Mockito.mock(RemoteAccessVpnDao.class); + manager._accountMgr = Mockito.mock(AccountManager.class); + manager._firewallMgr = Mockito.mock(FirewallManager.class); + RemoteAccessVpnVO vpn = Mockito.mock(RemoteAccessVpnVO.class); + Account caller = Mockito.mock(Account.class); + CallContext context = Mockito.mock(CallContext.class); + Mockito.when(vpn.getVpcId()).thenReturn(VPC_ID); + Mockito.when(manager._remoteAccessVpnDao.findByPublicIpAddress(31L)).thenReturn(vpn); + Mockito.when(manager.vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, + Network.Provider.VPCVirtualRouter)).thenReturn(false); + Mockito.when(context.getCallingAccount()).thenReturn(caller); + + try (MockedStatic callContext = Mockito.mockStatic(CallContext.class)) { + callContext.when(CallContext::current).thenReturn(context); + + InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.startRemoteAccessVpn(31L, false)); + + Assert.assertTrue(exception.getMessage().contains("does not implement the Remote Access VPN service")); + } + Mockito.verify(provider, Mockito.never()).startVpn(vpn); + Mockito.verify(manager._remoteAccessVpnDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void startRemoteAccessVpnFailsWhenMappedProviderDoesNotStartIt() throws ResourceUnavailableException { + RemoteAccessVPNServiceProvider provider = mockProvider(Network.Provider.VPCVirtualRouter); + RemoteAccessVpnManagerImpl manager = managerWithProvider(provider); + manager._remoteAccessVpnDao = Mockito.mock(RemoteAccessVpnDao.class); + manager._accountMgr = Mockito.mock(AccountManager.class); + manager._firewallMgr = Mockito.mock(FirewallManager.class); + RemoteAccessVpnVO vpn = Mockito.mock(RemoteAccessVpnVO.class); + Account caller = Mockito.mock(Account.class); + CallContext context = Mockito.mock(CallContext.class); + Mockito.when(vpn.getId()).thenReturn(22L); + Mockito.when(vpn.getVpcId()).thenReturn(VPC_ID); + Mockito.when(manager._remoteAccessVpnDao.findByPublicIpAddress(32L)).thenReturn(vpn); + Mockito.when(manager.vpcManager.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, + Network.Provider.VPCVirtualRouter)).thenReturn(true); + Mockito.when(context.getCallingAccount()).thenReturn(caller); + Mockito.when(provider.startVpn(vpn)).thenReturn(false); + Mockito.when(provider.getName()).thenReturn("VpcVirtualRouter"); + + try (MockedStatic callContext = Mockito.mockStatic(CallContext.class)) { + callContext.when(CallContext::current).thenReturn(context); + + ResourceUnavailableException exception = Assert.assertThrows(ResourceUnavailableException.class, + () -> manager.startRemoteAccessVpn(32L, false)); + + Assert.assertTrue(exception.getMessage().contains("Failed to start Remote Access VPN")); + } + Mockito.verify(manager._remoteAccessVpnDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + @Test public void validateValidateIpRangeRangeLengthLessThan2MustThrowException(){ String ipRange = "192.168.0.1"; diff --git a/server/src/test/java/com/cloud/network/vpn/Site2SiteVpnManagerImplTest.java b/server/src/test/java/com/cloud/network/vpn/Site2SiteVpnManagerImplTest.java index 291d3a4aa812..99dc75f2463f 100644 --- a/server/src/test/java/com/cloud/network/vpn/Site2SiteVpnManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpn/Site2SiteVpnManagerImplTest.java @@ -19,10 +19,12 @@ package com.cloud.network.vpn; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Site2SiteVpnConnection; import com.cloud.network.Site2SiteVpnConnection.State; import com.cloud.network.Site2SiteVpnGateway; +import com.cloud.network.Network; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.Site2SiteCustomerGatewayDao; @@ -32,6 +34,7 @@ import com.cloud.network.dao.Site2SiteVpnGatewayDao; import com.cloud.network.dao.Site2SiteVpnGatewayVO; import com.cloud.network.element.Site2SiteVpnServiceProvider; +import com.cloud.network.element.NetworkElement; import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; @@ -41,7 +44,9 @@ import com.cloud.user.AccountVO; import com.cloud.user.User; import com.cloud.user.UserVO; +import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; import com.cloud.vm.DomainRouterVO; import org.apache.cloudstack.acl.SecurityChecker; @@ -53,17 +58,21 @@ import org.apache.cloudstack.api.command.user.vpn.DeleteVpnCustomerGatewayCmd; import org.apache.cloudstack.api.command.user.vpn.DeleteVpnGatewayCmd; import org.apache.cloudstack.api.command.user.vpn.ResetVpnConnectionCmd; +import org.apache.cloudstack.api.command.user.vpn.UpdateVpnCustomerGatewayCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import java.lang.reflect.Field; import java.util.ArrayList; @@ -74,13 +83,16 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -161,10 +173,32 @@ public void setUp() throws Exception { when(customerGateway.getIkeVersion()).thenReturn("ike"); vpnConnection = new Site2SiteVpnConnectionVO(ACCOUNT_ID, DOMAIN_ID, VPN_GATEWAY_ID, CUSTOMER_GATEWAY_ID, false); + ReflectionTestUtils.setField(vpnConnection, "id", VPN_CONNECTION_ID); vpnConnection.setState(State.Pending); when(_accountMgr.getAccount(ACCOUNT_ID)).thenReturn(account); doNothing().when(_accountMgr).checkAccess(any(Account.class), nullable(SecurityChecker.AccessType.class), anyBoolean(), any()); + when(_s2sProviders.iterator()).thenReturn(List.of().iterator()); + } + + private Site2SiteVpnServiceProvider mockVpcVirtualRouterProvider() { + Site2SiteVpnServiceProvider provider = mock(Site2SiteVpnServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + when(((NetworkElement) provider).getProvider()).thenReturn(Network.Provider.VPCVirtualRouter); + when(_vpcMgr.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, Network.Provider.VPCVirtualRouter)) + .thenReturn(true); + when(_s2sProviders.iterator()).thenAnswer(invocation -> List.of(provider).iterator()); + return provider; + } + + private Site2SiteVpnServiceProvider mockNsxProvider() { + Site2SiteVpnServiceProvider provider = mock(Site2SiteVpnServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + when(((NetworkElement) provider).getProvider()).thenReturn(Network.Provider.Nsx); + when(_vpcMgr.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, Network.Provider.Nsx)) + .thenReturn(true); + when(_s2sProviders.iterator()).thenAnswer(invocation -> List.of(provider).iterator()); + return provider; } @After @@ -214,9 +248,11 @@ public void testCreateVpnGatewaySuccess() { when(cmd.getVpcId()).thenReturn(VPC_ID); when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); when(cmd.isDisplay()).thenReturn(true); + when(cmd.getIpAddressId()).thenReturn(null); when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + mockVpcVirtualRouterProvider(); when(_ipAddressDao.listByAssociatedVpc(VPC_ID, true)).thenReturn(List.of(ipAddress)); when(_vpnGatewayDao.persist(any(Site2SiteVpnGatewayVO.class))).thenReturn(vpnGateway); @@ -226,6 +262,48 @@ public void testCreateVpnGatewaySuccess() { verify(_vpnGatewayDao).persist(any(Site2SiteVpnGatewayVO.class)); } + @Test + public void testCreateVpnGatewayAcceptsValidProviderOwnedIp() { + CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); + when(cmd.getVpcId()).thenReturn(VPC_ID); + when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(cmd.getIpAddressId()).thenReturn(null); + when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + Site2SiteVpnServiceProvider provider = mockNsxProvider(); + when(provider.acquireVpnGatewayIp(vpc, null)).thenReturn(ipAddress); + when(_ipAddressDao.findById(IP_ADDRESS_ID)).thenReturn(ipAddress); + when(ipAddress.getAddress()).thenReturn(new Ip("203.0.113.10")); + when(ipAddress.readyToUse()).thenReturn(true); + when(_vpnGatewayDao.persist(any(Site2SiteVpnGatewayVO.class))).thenReturn(vpnGateway); + + Site2SiteVpnGateway result = site2SiteVpnManager.createVpnGateway(cmd); + + assertNotNull(result); + verify(provider, never()).releaseVpnGatewayIp(any(Site2SiteVpnGateway.class)); + } + + @Test + public void testCreateVpnGatewayRejectsProviderIpOutsideVpcAndCleansUp() { + CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); + when(cmd.getVpcId()).thenReturn(VPC_ID); + when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(cmd.getIpAddressId()).thenReturn(null); + when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + Site2SiteVpnServiceProvider provider = mockNsxProvider(); + when(provider.acquireVpnGatewayIp(vpc, null)).thenReturn(ipAddress); + when(_ipAddressDao.findById(IP_ADDRESS_ID)).thenReturn(ipAddress); + when(ipAddress.getAddress()).thenReturn(new Ip("203.0.113.10")); + when(ipAddress.readyToUse()).thenReturn(true); + when(ipAddress.getVpcId()).thenReturn(99L); + + assertThrows(CloudRuntimeException.class, () -> site2SiteVpnManager.createVpnGateway(cmd)); + + verify(provider).releaseVpnGatewayIp(any(Site2SiteVpnGateway.class)); + verify(_vpnGatewayDao, never()).persist(any(Site2SiteVpnGatewayVO.class)); + } + @Test(expected = InvalidParameterValueException.class) public void testCreateVpnGatewayInvalidVpc() { CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); @@ -254,14 +332,56 @@ public void testCreateVpnGatewayNoSourceNatIp() { CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); when(cmd.getVpcId()).thenReturn(VPC_ID); when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(cmd.getIpAddressId()).thenReturn(null); when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + mockVpcVirtualRouterProvider(); when(_ipAddressDao.listByAssociatedVpc(VPC_ID, true)).thenReturn(new ArrayList<>()); site2SiteVpnManager.createVpnGateway(cmd); } + @Test + public void testCreateVpnGatewaySkipsSystemVmSourceNatIp() { + CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); + when(cmd.getVpcId()).thenReturn(VPC_ID); + when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(cmd.getIpAddressId()).thenReturn(null); + when(cmd.isDisplay()).thenReturn(true); + IPAddressVO systemVmIp = mock(IPAddressVO.class); + when(systemVmIp.isForSystemVms()).thenReturn(true); + when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + mockVpcVirtualRouterProvider(); + when(_ipAddressDao.listByAssociatedVpc(VPC_ID, true)).thenReturn(List.of(systemVmIp, ipAddress)); + when(_vpnGatewayDao.persist(any(Site2SiteVpnGatewayVO.class))).thenReturn(vpnGateway); + + Site2SiteVpnGateway result = site2SiteVpnManager.createVpnGateway(cmd); + + assertNotNull(result); + ArgumentCaptor gatewayCaptor = ArgumentCaptor.forClass(Site2SiteVpnGatewayVO.class); + verify(_vpnGatewayDao).persist(gatewayCaptor.capture()); + assertEquals(IP_ADDRESS_ID.longValue(), gatewayCaptor.getValue().getAddrId()); + } + + @Test(expected = CloudRuntimeException.class) + public void testCreateVpnGatewayRejectsMultipleCustomerSourceNatIps() { + CreateVpnGatewayCmd cmd = mock(CreateVpnGatewayCmd.class); + when(cmd.getVpcId()).thenReturn(VPC_ID); + when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(cmd.getIpAddressId()).thenReturn(null); + IPAddressVO secondIp = mock(IPAddressVO.class); + when(ipAddress.getAddress()).thenReturn(new Ip("203.0.113.34")); + when(secondIp.getAddress()).thenReturn(new Ip("203.0.113.35")); + when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(null); + mockVpcVirtualRouterProvider(); + when(_ipAddressDao.listByAssociatedVpc(VPC_ID, true)).thenReturn(List.of(ipAddress, secondIp)); + + site2SiteVpnManager.createVpnGateway(cmd); + } + @Test(expected = InvalidParameterValueException.class) public void testCreateCustomerGatewayInvalidIp() { CreateVpnCustomerGatewayCmd cmd = mock(CreateVpnCustomerGatewayCmd.class); @@ -477,6 +597,7 @@ public void testCreateVpnConnectionCidrOverlapWithVpc() { when(_vpnConnectionDao.findByVpnGatewayIdAndCustomerGatewayId(VPN_GATEWAY_ID, CUSTOMER_GATEWAY_ID)).thenReturn(null); when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(vpnGateway); when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + mockVpcVirtualRouterProvider(); try (MockedStatic netUtilsMock = Mockito.mockStatic(NetUtils.class)) { netUtilsMock.when(() -> NetUtils.isNetworksOverlap("10.0.0.0/16", "10.0.0.0/24")).thenReturn(true); @@ -497,6 +618,7 @@ public void testCreateVpnConnectionExceedsLimit() { when(_vpnConnectionDao.findByVpnGatewayIdAndCustomerGatewayId(VPN_GATEWAY_ID, CUSTOMER_GATEWAY_ID)).thenReturn(null); when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(vpnGateway); when(_vpcDao.findById(VPC_ID)).thenReturn(vpc); + mockVpcVirtualRouterProvider(); List existingConns = new ArrayList<>(); for (int i = 0; i < 4; i++) { @@ -507,6 +629,109 @@ public void testCreateVpnConnectionExceedsLimit() { site2SiteVpnManager.createVpnConnection(cmd); } + @Test + public void testCreateVpnConnectionValidatesCustomerGatewayWithOwningProviderBeforePersist() { + CreateVpnConnectionCmd cmd = mock(CreateVpnConnectionCmd.class); + when(cmd.getVpnGatewayId()).thenReturn(VPN_GATEWAY_ID); + when(cmd.getCustomerGatewayId()).thenReturn(CUSTOMER_GATEWAY_ID); + when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); + when(_customerGatewayDao.findById(CUSTOMER_GATEWAY_ID)).thenReturn(customerGateway); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + when(_vpnConnectionDao.findByVpnGatewayIdAndCustomerGatewayId(VPN_GATEWAY_ID, CUSTOMER_GATEWAY_ID)).thenReturn(null); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockNsxProvider(); + when(provider.ownsVpnGateway(vpnGateway)).thenReturn(true); + Mockito.doThrow(new InvalidParameterValueException("unsupported by provider")) + .when(provider).validateSite2SiteVpnCustomerGateway(customerGateway); + + assertThrows(InvalidParameterValueException.class, () -> site2SiteVpnManager.createVpnConnection(cmd)); + + verify(_vpnConnectionDao, never()).persist(any(Site2SiteVpnConnectionVO.class)); + } + + @Test + public void testUpdateCustomerGatewayValidatesProposedValuesBeforePersist() { + UpdateVpnCustomerGatewayCmd cmd = mock(UpdateVpnCustomerGatewayCmd.class); + when(cmd.getId()).thenReturn(CUSTOMER_GATEWAY_ID); + when(cmd.getGatewayIp()).thenReturn("203.0.113.10"); + when(cmd.getGuestCidrList()).thenReturn("192.168.100.0/24"); + when(cmd.getIpsecPsk()).thenReturn("presharedkey"); + when(cmd.getIkePolicy()).thenReturn("aes256-sha256;modp2048"); + when(cmd.getEspPolicy()).thenReturn("aes256-sha256;modp2048"); + when(cmd.getIkeVersion()).thenReturn("ikev2"); + when(_customerGatewayDao.findById(CUSTOMER_GATEWAY_ID)).thenReturn(customerGateway); + when(_vpnConnectionDao.listByCustomerGatewayId(CUSTOMER_GATEWAY_ID)).thenReturn(List.of(vpnConnection)); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockNsxProvider(); + when(provider.ownsVpnGateway(vpnGateway)).thenReturn(true); + Mockito.doThrow(new InvalidParameterValueException("unsupported by provider")) + .when(provider).validateSite2SiteVpnCustomerGateway(any(Site2SiteCustomerGatewayVO.class)); + + try (MockedStatic netUtilsMock = Mockito.mockStatic(NetUtils.class)) { + netUtilsMock.when(() -> NetUtils.isValidIp4("203.0.113.10")).thenReturn(true); + netUtilsMock.when(() -> NetUtils.isValidCidrList("192.168.100.0/24")).thenReturn(true); + netUtilsMock.when(() -> NetUtils.getCleanIp4CidrList("192.168.100.0/24")) + .thenReturn("192.168.100.0/24"); + netUtilsMock.when(() -> NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp2048")).thenReturn(true); + netUtilsMock.when(() -> NetUtils.isValidS2SVpnPolicy("esp", "aes256-sha256;modp2048")).thenReturn(true); + + assertThrows(InvalidParameterValueException.class, + () -> site2SiteVpnManager.updateCustomerGateway(cmd)); + } + + verify(_customerGatewayDao, never()).persist(customerGateway); + } + + @Test + public void testCustomerGatewayUpdateRestartUsesSingleConnectionLock() throws ResourceUnavailableException { + vpnConnection.setState(State.Connected); + when(_vpnConnectionDao.listByCustomerGatewayId(CUSTOMER_GATEWAY_ID)).thenReturn(List.of(vpnConnection)); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(VPC_ID, false)).thenReturn(true); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.stopSite2SiteVpn(vpnConnection)).thenReturn(true); + when(provider.startSite2SiteVpn(vpnConnection)).thenReturn(true); + + ReflectionTestUtils.invokeMethod(site2SiteVpnManager, "setupVpnConnection", account, CUSTOMER_GATEWAY_ID); + + verify(_vpnConnectionDao, times(1)).acquireInLockTable(VPN_CONNECTION_ID); + InOrder lifecycle = inOrder(provider, _vpcMgr, _vpnConnectionDao); + lifecycle.verify(provider).stopSite2SiteVpn(vpnConnection); + lifecycle.verify(_vpcMgr).applyStaticRouteForVpcVpnIfNeeded(VPC_ID, false); + lifecycle.verify(provider).startSite2SiteVpn(vpnConnection); + lifecycle.verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testUpdateCustomerGatewayDefinesDatabaseContextForConnectionLock() throws NoSuchMethodException { + assertTrue(Site2SiteVpnManagerImpl.class.getMethod("updateCustomerGateway", UpdateVpnCustomerGatewayCmd.class) + .isAnnotationPresent(DB.class)); + } + + @Test + public void testCustomerGatewayUpdateLockFailureIsReported() { + when(_vpnConnectionDao.listByCustomerGatewayId(CUSTOMER_GATEWAY_ID)).thenReturn(List.of(vpnConnection)); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(null); + + assertThrows(CloudRuntimeException.class, + () -> ReflectionTestUtils.invokeMethod(site2SiteVpnManager, "setupVpnConnection", account, CUSTOMER_GATEWAY_ID)); + + verify(_vpnConnectionDao, never()).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testCustomerGatewayUpdateDoesNotRestartPendingConnection() { + vpnConnection.setState(State.Pending); + when(_vpnConnectionDao.listByCustomerGatewayId(CUSTOMER_GATEWAY_ID)).thenReturn(List.of(vpnConnection)); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + + ReflectionTestUtils.invokeMethod(site2SiteVpnManager, "setupVpnConnection", account, CUSTOMER_GATEWAY_ID); + + verify(_vpnGatewayDao, never()).findById(anyLong()); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + @Test public void testDeleteCustomerGatewaySuccess() { DeleteVpnCustomerGatewayCmd cmd = mock(DeleteVpnCustomerGatewayCmd.class); @@ -539,6 +764,7 @@ public void testDeleteVpnGatewaySuccess() { when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); when(_vpnConnectionDao.listByVpnGatewayId(VPN_GATEWAY_ID)).thenReturn(new ArrayList<>()); + mockVpcVirtualRouterProvider(); boolean result = site2SiteVpnManager.deleteVpnGateway(cmd); @@ -562,15 +788,54 @@ public void testDeleteVpnConnectionSuccess() throws ResourceUnavailableException DeleteVpnConnectionCmd cmd = mock(DeleteVpnConnectionCmd.class); when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); - when(_vpnConnectionDao.findById(VPN_CONNECTION_ID)).thenReturn(vpnConnection); vpnConnection.setState(State.Pending); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(anyLong(), anyBoolean())).thenReturn(true); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.deleteSite2SiteVpn(any(Site2SiteVpnConnection.class))).thenReturn(true); boolean result = site2SiteVpnManager.deleteVpnConnection(cmd); assertTrue(result); - verify(_vpnConnectionDao).remove(VPN_CONNECTION_ID); + assertEquals(State.Removed, vpnConnection.getState()); + verify(_vpnConnectionDao, times(1)).acquireInLockTable(VPN_CONNECTION_ID); + InOrder lifecycle = inOrder(provider, _vpnConnectionDao, _vpcMgr); + lifecycle.verify(provider).deleteSite2SiteVpn(vpnConnection); + lifecycle.verify(_vpnConnectionDao).update(VPN_CONNECTION_ID, vpnConnection); + lifecycle.verify(_vpcMgr).applyStaticRouteForVpcVpnIfNeeded(VPC_ID, false); + lifecycle.verify(_vpnConnectionDao).remove(VPN_CONNECTION_ID); + lifecycle.verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testDeleteVpnConnectionProviderFailureKeepsRowAndReleasesLock() throws ResourceUnavailableException { + DeleteVpnConnectionCmd cmd = mock(DeleteVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + vpnConnection.setState(State.Connected); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.deleteSite2SiteVpn(vpnConnection)).thenReturn(false); + + assertThrows(ResourceUnavailableException.class, + () -> site2SiteVpnManager.deleteVpnConnection(cmd)); + + assertEquals(State.Error, vpnConnection.getState()); + verify(_vpnConnectionDao, never()).update(VPN_CONNECTION_ID, vpnConnection); + verify(_vpnConnectionDao, never()).remove(VPN_CONNECTION_ID); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testDeleteVpnConnectionLockFailureIsNotReportedAsMissing() { + DeleteVpnConnectionCmd cmd = mock(DeleteVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(null); + when(_vpnConnectionDao.findById(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + + assertThrows(CloudRuntimeException.class, + () -> site2SiteVpnManager.deleteVpnConnection(cmd)); } @Test @@ -578,9 +843,8 @@ public void testStartVpnConnectionSuccess() throws ResourceUnavailableException when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); vpnConnection.setState(State.Pending); when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); - Site2SiteVpnServiceProvider provider = mock(Site2SiteVpnServiceProvider.class); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); when(provider.startSite2SiteVpn(any(Site2SiteVpnConnection.class))).thenReturn(true); - when(_s2sProviders.iterator()).thenReturn(List.of(provider).iterator()); when(_vpnConnectionDao.persist(any(Site2SiteVpnConnectionVO.class))).thenReturn(vpnConnection); when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(anyLong(), anyBoolean())).thenReturn(true); @@ -590,6 +854,81 @@ public void testStartVpnConnectionSuccess() throws ResourceUnavailableException verify(_vpnConnectionDao, org.mockito.Mockito.atLeastOnce()).persist(any(Site2SiteVpnConnectionVO.class)); } + @Test + public void testStartVpnConnectionProviderFailureLeavesConnectionInError() throws ResourceUnavailableException { + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + vpnConnection.setState(State.Pending); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.startSite2SiteVpn(any(Site2SiteVpnConnection.class))) + .thenThrow(new InvalidParameterValueException("unsupported VPN policy")); + when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(anyLong(), anyBoolean())).thenReturn(true); + + assertThrows(InvalidParameterValueException.class, + () -> site2SiteVpnManager.startVpnConnection(VPN_CONNECTION_ID)); + + assertEquals(State.Error, vpnConnection.getState()); + verify(_vpnConnectionDao, org.mockito.Mockito.atLeastOnce()).persist(vpnConnection); + } + + @Test + public void testStartVpnConnectionMissingGatewayLeavesConnectionInError() { + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + vpnConnection.setState(State.Pending); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(null); + + assertThrows(CloudRuntimeException.class, + () -> site2SiteVpnManager.startVpnConnection(VPN_CONNECTION_ID)); + + assertEquals(State.Error, vpnConnection.getState()); + verify(_vpnConnectionDao, times(2)).persist(vpnConnection); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testGatewayLifecycleUsesPersistedProviderWhenOfferingChanges() throws ResourceUnavailableException { + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider originalProvider = mockVpcVirtualRouterProvider(); + Site2SiteVpnServiceProvider replacementProvider = mock(Site2SiteVpnServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + when(((NetworkElement) replacementProvider).getProvider()).thenReturn(Network.Provider.Nsx); + when(_vpcMgr.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, Network.Provider.Nsx)) + .thenReturn(true); + when(originalProvider.ownsVpnGateway(vpnGateway)).thenReturn(true); + when(originalProvider.startSite2SiteVpn(vpnConnection)).thenReturn(true); + when(_vpnConnectionDao.persist(any(Site2SiteVpnConnectionVO.class))).thenReturn(vpnConnection); + + site2SiteVpnManager.startVpnConnection(VPN_CONNECTION_ID); + + verify(originalProvider).startSite2SiteVpn(vpnConnection); + verify(replacementProvider, never()).startSite2SiteVpn(any(Site2SiteVpnConnection.class)); + } + + @Test + public void testUnmarkedGatewayUsesLegacyVirtualRouterWhenOfferingNowUsesNsx() throws ResourceUnavailableException { + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider nsxProvider = mock(Site2SiteVpnServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + Site2SiteVpnServiceProvider virtualRouterProvider = mock(Site2SiteVpnServiceProvider.class, + Mockito.withSettings().extraInterfaces(NetworkElement.class)); + when(((NetworkElement) nsxProvider).getProvider()).thenReturn(Network.Provider.Nsx); + when(((NetworkElement) virtualRouterProvider).getProvider()).thenReturn(Network.Provider.VPCVirtualRouter); + when(_vpcMgr.isProviderSupportServiceInVpc(VPC_ID, Network.Service.Vpn, Network.Provider.Nsx)) + .thenReturn(true); + when(nsxProvider.ownsVpnGateway(vpnGateway)).thenReturn(false); + when(virtualRouterProvider.ownsVpnGateway(vpnGateway)).thenReturn(false); + when(_s2sProviders.iterator()).thenAnswer(invocation -> List.of(nsxProvider, virtualRouterProvider).iterator()); + when(virtualRouterProvider.startSite2SiteVpn(vpnConnection)).thenReturn(true); + when(_vpnConnectionDao.persist(any(Site2SiteVpnConnectionVO.class))).thenReturn(vpnConnection); + + site2SiteVpnManager.startVpnConnection(VPN_CONNECTION_ID); + + verify(virtualRouterProvider).startSite2SiteVpn(vpnConnection); + verify(nsxProvider, never()).startSite2SiteVpn(any(Site2SiteVpnConnection.class)); + } + @Test(expected = InvalidParameterValueException.class) public void testStartVpnConnectionWrongState() throws ResourceUnavailableException { when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); @@ -603,20 +942,108 @@ public void testResetVpnConnectionSuccess() throws ResourceUnavailableException ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); - when(_vpnConnectionDao.findById(VPN_CONNECTION_ID)).thenReturn(vpnConnection); vpnConnection.setState(State.Connected); when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); - Site2SiteVpnServiceProvider provider = mock(Site2SiteVpnServiceProvider.class); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); when(provider.stopSite2SiteVpn(any(Site2SiteVpnConnection.class))).thenReturn(true); when(provider.startSite2SiteVpn(any(Site2SiteVpnConnection.class))).thenReturn(true); - when(_s2sProviders.iterator()).thenReturn(List.of(provider).iterator()); when(_vpnConnectionDao.persist(any(Site2SiteVpnConnectionVO.class))).thenReturn(vpnConnection); when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(anyLong(), anyBoolean())).thenReturn(true); Site2SiteVpnConnection result = site2SiteVpnManager.resetVpnConnection(cmd); assertNotNull(result); + assertEquals(State.Connecting, result.getState()); + verify(_vpnConnectionDao, times(1)).acquireInLockTable(VPN_CONNECTION_ID); + InOrder lifecycle = inOrder(provider, _vpnConnectionDao, _vpcMgr); + lifecycle.verify(provider).stopSite2SiteVpn(vpnConnection); + lifecycle.verify(_vpcMgr).applyStaticRouteForVpcVpnIfNeeded(VPC_ID, false); + lifecycle.verify(provider).startSite2SiteVpn(vpnConnection); + lifecycle.verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testResetVpnConnectionStartFailureLeavesErrorAndReleasesLock() throws ResourceUnavailableException { + ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + vpnConnection.setState(State.Connected); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.stopSite2SiteVpn(vpnConnection)).thenReturn(true); + when(provider.startSite2SiteVpn(vpnConnection)).thenReturn(false); + when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(VPC_ID, false)).thenReturn(true); + + assertThrows(ResourceUnavailableException.class, + () -> site2SiteVpnManager.resetVpnConnection(cmd)); + + assertEquals(State.Error, vpnConnection.getState()); + verify(_vpnConnectionDao, times(1)).acquireInLockTable(VPN_CONNECTION_ID); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testResetVpnConnectionAllowsPendingConnection() throws ResourceUnavailableException { + ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + vpnConnection.setState(State.Pending); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.stopSite2SiteVpn(vpnConnection)).thenReturn(true); + when(provider.startSite2SiteVpn(vpnConnection)).thenReturn(true); + + Site2SiteVpnConnection result = site2SiteVpnManager.resetVpnConnection(cmd); + + assertEquals(State.Connecting, result.getState()); + verify(provider).stopSite2SiteVpn(vpnConnection); + verify(provider).startSite2SiteVpn(vpnConnection); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testResetVpnConnectionStopFailureDoesNotStartAndReleasesLock() throws ResourceUnavailableException { + ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + vpnConnection.setState(State.Connected); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); + when(provider.stopSite2SiteVpn(vpnConnection)).thenReturn(false); + + assertThrows(ResourceUnavailableException.class, + () -> site2SiteVpnManager.resetVpnConnection(cmd)); + + assertEquals(State.Error, vpnConnection.getState()); + verify(provider, never()).startSite2SiteVpn(vpnConnection); + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + } + + @Test + public void testResetVpnConnectionAccessDeniedReleasesLock() { + ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + Mockito.doThrow(new PermissionDeniedException("denied")) + .when(_accountMgr).checkAccess(account, null, false, vpnConnection); + + assertThrows(PermissionDeniedException.class, + () -> site2SiteVpnManager.resetVpnConnection(cmd)); + + verify(_vpnConnectionDao).releaseFromLockTable(VPN_CONNECTION_ID); + verify(_vpnGatewayDao, never()).findById(anyLong()); + } + + @Test + public void testResetVpnConnectionLockFailureIsNotReportedAsMissing() { + ResetVpnConnectionCmd cmd = mock(ResetVpnConnectionCmd.class); + when(cmd.getId()).thenReturn(VPN_CONNECTION_ID); + when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(null); + when(_vpnConnectionDao.findById(VPN_CONNECTION_ID)).thenReturn(vpnConnection); + + assertThrows(CloudRuntimeException.class, + () -> site2SiteVpnManager.resetVpnConnection(cmd)); } @Test @@ -633,6 +1060,7 @@ public void testCleanupVpnConnectionByVpc() { public void testCleanupVpnGatewayByVpc() { when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(vpnGateway); when(_vpnConnectionDao.listByVpnGatewayId(VPN_GATEWAY_ID)).thenReturn(new ArrayList<>()); + mockVpcVirtualRouterProvider(); boolean result = site2SiteVpnManager.cleanupVpnGatewayByVpc(VPC_ID); @@ -654,6 +1082,8 @@ public void testCleanupVpnGatewayByVpcNotFound() { public void testGetConnectionsForRouter() { DomainRouterVO router = mock(DomainRouterVO.class); when(router.getVpcId()).thenReturn(VPC_ID); + when(_vpnGatewayDao.findByVpcId(VPC_ID)).thenReturn(vpnGateway); + mockVpcVirtualRouterProvider(); when(_vpnConnectionDao.listByVpcId(VPC_ID)).thenReturn(List.of(vpnConnection)); List result = site2SiteVpnManager.getConnectionsForRouter(router); @@ -695,9 +1125,8 @@ public void testReconnectDisconnectedVpnByVpc() throws ResourceUnavailableExcept when(_customerGatewayDao.findById(CUSTOMER_GATEWAY_ID)).thenReturn(customerGateway); when(_vpnConnectionDao.acquireInLockTable(VPN_CONNECTION_ID)).thenReturn(conn); when(_vpnGatewayDao.findById(VPN_GATEWAY_ID)).thenReturn(vpnGateway); - Site2SiteVpnServiceProvider provider = mock(Site2SiteVpnServiceProvider.class); + Site2SiteVpnServiceProvider provider = mockVpcVirtualRouterProvider(); when(provider.startSite2SiteVpn(any(Site2SiteVpnConnection.class))).thenReturn(true); - when(_s2sProviders.iterator()).thenReturn(List.of(provider).iterator()); when(_vpnConnectionDao.persist(any(Site2SiteVpnConnectionVO.class))).thenReturn(conn); when(_vpcMgr.applyStaticRouteForVpcVpnIfNeeded(anyLong(), anyBoolean())).thenReturn(true); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 1239dd23e7ec..2404251c5c6d 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -180,6 +180,13 @@ public IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Boolea return null; } + @Override + public IpAddress allocateIP(Account ipOwner, long zoneId, Long networkId, Long vpcId, Boolean displayIp, + String ipaddress) throws ResourceAllocationException, InsufficientAddressCapacityException, + ConcurrentOperationException { + return null; + } + @Override public IpAddress reserveIpAddress(Account account, Boolean displayIp, Long ipAddressId) throws ResourceAllocationException { return null; diff --git a/server/src/test/java/com/cloud/vpc/dao/MockVpcDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockVpcDaoImpl.java index 76403be76222..5cdda7e9f166 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockVpcDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockVpcDaoImpl.java @@ -60,6 +60,11 @@ public List listByAccountId(long accountId) { return null; } + @Override + public List listByZone(long zoneId) { + return null; + } + /* (non-Javadoc) * @see com.cloud.network.vpc.Dao.VpcDao#listInactiveVpcs() */ diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index c99328fff9ff..3f01729a9d90 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -100,6 +100,7 @@ 'listNsxControllers': 'NSX', 'addNsxController': 'NSX', 'deleteNsxController': 'NSX', + 'NsxVrfGateway': 'NSX', 'NetrisProvider': 'Netris', 'Vpn': 'VPN', 'Limit': 'Resource Limit', diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 775de26103a0..d0e37c81c451 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1880,6 +1880,9 @@ "label.nsx.provider.edgecluster": "NSX provider edge Cluster", "label.nsx.provider.tier0gateway": "NSX provider tier-0 gateway", "label.nsx.provider.transportzone": "NSX provider transport Zone", +"label.nsx.ip.discovery.profile.id": "NSX IP discovery profile ID", +"label.nsx.mac.discovery.profile.id": "NSX MAC discovery profile ID", +"label.nsx.segment.security.profile.id": "NSX segment security profile ID", "label.nsx.supports.internal.lb": "Enable NSX internal LB service", "label.nsx.supports.lb": "Enable NSX LB service", "label.num.cpu.cores": "# of CPU cores", @@ -3825,6 +3828,7 @@ "message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster", "message.kubernetes.cluster.delete": "Please confirm that you want to destroy the Cluster.", "message.kubeconfig.cluster.not.available": "Kubernetes Cluster kubeconfig not available currently.", +"message.kubernetes.cluster.network.rules.reconcile": "Please confirm that you want to recreate any missing CloudStack-managed network rules for this Kubernetes cluster. User-managed rules will not be deleted or changed.", "message.kubernetes.cluster.remove.nodes": "Please confirm that you want to remove the following nodes from the cluster", "message.kubernetes.cluster.scale": "Please select desired Cluster configuration.", "message.kubernetes.cluster.start": "Please confirm that you want to start the Cluster.", @@ -3891,6 +3895,9 @@ "message.network.offering.mac.address.changes": "Applicable for guest Networks on VMware hypervisor only.\nReject - If the guest OS changes the effective MAC address of the Instance to a value that is different from the MAC address of the Instance Network adapter (set in the .vmx configuration file), the switch drops all inbound frames to the adapter.\nIf the guest OS changes the effective MAC address of the Instance back to the MAC address of the Instance Network adapter, the virtual machine receives frames again.\nAccept - If the guest OS changes the effective MAC address of the virtual machine to a value that is different from the MAC address of the Instance Network adapter, the switch allows frames to the new address to pass.\nNone - Default to value from global setting.", "message.network.offering.mac.learning": "Applicable for guest Networks on VMware hypervisor only with VMware Distributed Virtual Switches version 6.6.0 & above and vSphere version 6.7 & above.\nMAC learning enables Network connectivity for multiple MAC addresses behind a single vNIC.\nNone - Default to value from global setting.", "message.network.offering.mac.learning.warning": "WARNING: In order to use MAC Learning you must ensure your hypervisor hosts are running ESXi 6.7+ and the Network uses distributed vSwitch 6.6.0+.", +"message.network.offering.nsx.ip.discovery.profile.id": "ID of an existing NSX IP discovery profile to bind to segments created from this offering.", +"message.network.offering.nsx.mac.discovery.profile.id": "ID of an existing NSX MAC discovery profile to bind to segments created from this offering.", +"message.network.offering.nsx.segment.security.profile.id": "ID of an existing NSX segment security profile to bind to segments created from this offering.", "message.network.offering.promiscuous.mode": "Applicable for guest Networks on VMware hypervisor only.\nReject - The switch drops any outbound frame from a virtual machine adapter with a source MAC address that is different from the one in the .vmx configuration file.\nAccept - The switch does not perform filtering, and permits all outbound frames.\nNone - Default to value from global setting.", "message.network.removenic": "Please confirm that want to remove this NIC, which will also remove the associated Network from the Instance.", "message.network.restart.required": "Restart is required for network(s). Click here to view network(s) which require restart.", @@ -3968,6 +3975,9 @@ "message.remove.sticky.policy.failed": "Failed to remove sticky policy.", "message.remove.sticky.policy.processing": "Removing sticky policy...", "message.remove.vpc": "Please confirm that you want to remove the VPC", +"message.replace.acl.failed": "Failed to replace the Network ACL list", +"message.replace.acl.processing": "Replacing the Network ACL list...", +"message.replace.acl.success": "Successfully replaced the Network ACL list", "message.request.failed": "Request failed.", "message.request.no.data": "There is no data to show.", "message.required.add.least.ip": "Please add at least 1 IP Range", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index b6e83b5eb438..d9f1ad76a0d1 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -824,6 +824,17 @@ export default { popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleKubernetesCluster.vue'))) }, + { + api: 'reconcileKubernetesClusterNetworkRules', + icon: 'reload-outlined', + label: 'label.reset.to.default', + message: 'message.kubernetes.cluster.network.rules.reconcile', + dataView: true, + show: (record) => { return ['Running'].includes(record.state) && record.clustertype === 'CloudManaged' }, + groupAction: true, + popup: true, + groupMap: (selection) => { return selection.map(x => { return { id: x } }) } + }, { api: 'updateKubernetesClusterAffinityGroups', icon: 'swap-outlined', diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index b8856c7c9b00..8f8e055bfbf9 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -217,7 +217,6 @@ export const pollJobPlugin = { export const notifierPlugin = { install (app) { app.config.globalProperties.$notifyError = function (error) { - console.log(error) var msg = i18n.global.t('message.request.failed') var desc = '' if (error && error.response) { diff --git a/ui/src/utils/request.js b/ui/src/utils/request.js index 2317aac04465..8d16dbe8aa59 100644 --- a/ui/src/utils/request.js +++ b/ui/src/utils/request.js @@ -34,7 +34,6 @@ const err = (error) => { const response = error.response let countNotify = store.getters.countNotify if (response) { - console.log(response) if (response.status === 403) { const data = response.data countNotify++ diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index 5410a9b9502f..8c206ffb9d07 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -87,7 +87,7 @@ - +