Skip to content

Commit efe00aa

Browse files
nvazqueznvazquez
authored
[KVM] Rolling maintenance (apache#3610)
1 parent 016644d commit efe00aa

43 files changed

Lines changed: 2600 additions & 12 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎agent/bindir/rolling-maintenance.in‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/python
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing,
11+
# software distributed under the License is distributed on an
12+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
# KIND, either express or implied. See the License for the
14+
# specific language governing permissions and limitations
15+
# under the License.
16+
17+
from subprocess import *
18+
import sys
19+
import logging
20+
21+
LOG_FILE='/var/log/cloudstack/agent/rolling-maintenance.log'
22+
AVOID_MAINTENANCE_EXIT_STATUS=70
23+
24+
logging.basicConfig(filename=LOG_FILE,
25+
filemode='a',
26+
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
27+
datefmt='%H:%M:%S',
28+
level=logging.INFO)
29+
logger = logging.getLogger('rolling-maintenance')
30+
31+
32+
def execute_script(stage, script, payload, timeout):
33+
logger.info("Executing script: %s for stage: %s" % (script, stage))
34+
35+
try:
36+
command = "timeout %s %s " % (str(timeout), script)
37+
if payload:
38+
logger.info("Adding payload: %s" % payload)
39+
command += " " + payload
40+
pout = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
41+
exitStatus = pout.wait()
42+
stdout, stderr = pout.communicate()
43+
44+
success = True if exitStatus == 0 or exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
45+
avoid_maintenance = True if exitStatus == AVOID_MAINTENANCE_EXIT_STATUS else False
46+
return {"success": success, "message": stdout.strip(), "avoidmaintenance": avoid_maintenance}
47+
except Exception as e:
48+
logger.error("Error in stage %s: %s" % (script, e))
49+
sys.exit(1)
50+
51+
52+
if __name__ == '__main__':
53+
try:
54+
logger.info(sys.argv)
55+
if len(sys.argv) < 2:
56+
logger.error("Arguments missing")
57+
sys.exit(0)
58+
59+
args = sys.argv[1]
60+
params = args.split(',')
61+
if len(params) < 5:
62+
logger.error("Wrong number of parameters received, STAGE,SCRIPT,TIMEOUT,RESULTS_FILE,OUTPUT_FILE"
63+
"[,PAYLOAD] expected")
64+
sys.exit(0)
65+
66+
stage = params[0]
67+
script = params[1]
68+
timeout = params[2]
69+
results_file_path = params[3]
70+
output_file_path = params[4]
71+
payload = params[5] if len(params) > 5 else None
72+
logger.info("Received parameters: stage: %s, script: %s, timeout: %s, results_file: %s, output_file: %s "
73+
"and payload: %s" % (stage, script, timeout, results_file_path, output_file_path, payload))
74+
75+
results = execute_script(stage, script, payload, timeout)
76+
77+
# Persist results and output on a file
78+
output_file = open(output_file_path, "w+")
79+
output_file.write(results['message'])
80+
output_file.close()
81+
82+
results_file = open(results_file_path, "w+")
83+
results_file.write("%s,%s,%s" % (stage, str(results['success']), str(results['avoidmaintenance'])))
84+
results_file.close()
85+
86+
msg = "Successful execution of %s" if results['success'] else "Script execution failed: %s"
87+
logger.info(results['message'])
88+
logger.info(msg % script)
89+
except Exception as e:
90+
logger.error("Unexpected error on systemd service: %s" % e)
91+
sys.exit(1)

‎agent/conf/agent.properties‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ hypervisor.type=kvm
118118
# This parameter specifies a directory on the host local storage for temporary storing direct download templates
119119
#direct.download.temporary.download.location=/var/lib/libvirt/images
120120

121+
# set the rolling maintenance hook scripts directory
122+
#rolling.maintenance.hooks.dir=/etc/cloudstack/agent/hooks.d
123+
124+
# disable the rolling maintenance service execution
125+
#rolling.maintenance.service.executor.disabled=true
126+
121127
# set the hypervisor URI. Usually there is no need for changing this
122128
# For KVM: qemu:///system
123129
# For LXC: lxc:///

‎agent/conf/cloudstack-agent.logrotate.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log {
18+
/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log {
1919
copytruncate
2020
daily
2121
rotate 5

‎api/src/main/java/com/cloud/deploy/DataCenterDeployment.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public class DataCenterDeployment implements DeploymentPlan {
3333
boolean _recreateDisks;
3434
ReservationContext _context;
3535
List<Long> preferredHostIds = new ArrayList<>();
36+
boolean migrationPlan;
3637

3738
public DataCenterDeployment(long dataCenterId) {
3839
this(dataCenterId, null, null, null, null, null);
@@ -107,4 +108,13 @@ public List<Long> getPreferredHosts() {
107108
return this.preferredHostIds;
108109
}
109110

111+
public void setMigrationPlan(boolean migrationPlan) {
112+
this.migrationPlan = migrationPlan;
113+
}
114+
115+
@Override
116+
public boolean isMigrationPlan() {
117+
return migrationPlan;
118+
}
119+
110120
}

‎api/src/main/java/com/cloud/deploy/DeploymentPlan.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,6 @@ public interface DeploymentPlan {
7171
void setPreferredHosts(List<Long> hostIds);
7272

7373
List<Long> getPreferredHosts();
74+
75+
boolean isMigrationPlan();
7476
}

‎api/src/main/java/com/cloud/event/EventTypes.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@
7676
import com.cloud.vm.Nic;
7777
import com.cloud.vm.NicSecondaryIp;
7878
import com.cloud.vm.VirtualMachine;
79+
import org.apache.cloudstack.api.response.ClusterResponse;
80+
import org.apache.cloudstack.api.response.HostResponse;
81+
import org.apache.cloudstack.api.response.PodResponse;
82+
import org.apache.cloudstack.api.response.ZoneResponse;
7983

8084
public class EventTypes {
8185

@@ -591,6 +595,13 @@ public class EventTypes {
591595
// Diagnostics Events
592596
public static final String EVENT_SYSTEM_VM_DIAGNOSTICS = "SYSTEM.VM.DIAGNOSTICS";
593597

598+
// Rolling Maintenance
599+
public static final String EVENT_START_ROLLING_MAINTENANCE = "SYSTEM.ROLLING.MAINTENANCE";
600+
public static final String EVENT_HOST_ROLLING_MAINTENANCE = "HOST.ROLLING.MAINTENANCE";
601+
public static final String EVENT_CLUSTER_ROLLING_MAINTENANCE = "CLUSTER.ROLLING.MAINTENANCE";
602+
public static final String EVENT_POD_ROLLING_MAINTENANCE = "POD.ROLLING.MAINTENANCE";
603+
public static final String EVENT_ZONE_ROLLING_MAINTENANCE = "ZONE.ROLLING.MAINTENANCE";
604+
594605
static {
595606

596607
// TODO: need a way to force author adding event types to declare the entity details as well, with out braking
@@ -990,6 +1001,11 @@ public class EventTypes {
9901001
entityEventDetails.put(EVENT_TEMPLATE_DIRECT_DOWNLOAD_FAILURE, VirtualMachineTemplate.class);
9911002
entityEventDetails.put(EVENT_ISO_DIRECT_DOWNLOAD_FAILURE, "Iso");
9921003
entityEventDetails.put(EVENT_SYSTEM_VM_DIAGNOSTICS, VirtualMachine.class);
1004+
1005+
entityEventDetails.put(EVENT_ZONE_ROLLING_MAINTENANCE, ZoneResponse.class);
1006+
entityEventDetails.put(EVENT_POD_ROLLING_MAINTENANCE, PodResponse.class);
1007+
entityEventDetails.put(EVENT_CLUSTER_ROLLING_MAINTENANCE, ClusterResponse.class);
1008+
entityEventDetails.put(EVENT_HOST_ROLLING_MAINTENANCE, HostResponse.class);
9931009
}
9941010

9951011
public static String getEntityForEvent(String eventName) {

‎api/src/main/java/com/cloud/resource/RollingMaintenanceManager.java‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.resource;
18+
19+
import com.cloud.host.Host;
20+
import com.cloud.utils.Pair;
21+
import com.cloud.utils.Ternary;
22+
import com.cloud.utils.exception.CloudRuntimeException;
23+
import org.apache.cloudstack.api.command.admin.resource.StartRollingMaintenanceCmd;
24+
import org.apache.cloudstack.framework.config.ConfigKey;
25+
import org.apache.cloudstack.framework.config.Configurable;
26+
27+
import java.util.Date;
28+
import java.util.List;
29+
30+
public interface RollingMaintenanceManager extends Configurable {
31+
32+
ConfigKey<Integer> KvmRollingMaintenanceStageTimeout = new ConfigKey<>("Advanced", Integer.class,
33+
"kvm.rolling.maintenance.stage.timeout", "1800",
34+
"Wait timeout (in seconds) for a rolling maintenance stage update from hosts",
35+
true, ConfigKey.Scope.Global);
36+
ConfigKey<Integer> KvmRollingMaintenancePingInterval = new ConfigKey<>("Advanced", Integer.class,
37+
"kvm.rolling.maintenance.ping.interval", "10",
38+
"Ping interval in seconds between management server and hosts performing stages during rolling maintenance",
39+
true, ConfigKey.Scope.Global);
40+
ConfigKey<Integer> KvmRollingMaintenanceWaitForMaintenanceTimeout = new ConfigKey<>("Advanced", Integer.class,
41+
"kvm.rolling.maintenance.wait.maintenance.timeout", "1800",
42+
"Timeout (in seconds) to wait for a host preparing to enter maintenance mode",
43+
true, ConfigKey.Scope.Global);
44+
45+
class HostSkipped {
46+
private Host host;
47+
private String reason;
48+
49+
public HostSkipped(Host host, String reason) {
50+
this.host = host;
51+
this.reason = reason;
52+
}
53+
54+
public Host getHost() {
55+
return host;
56+
}
57+
58+
public void setHost(Host host) {
59+
this.host = host;
60+
}
61+
62+
public String getReason() {
63+
return reason;
64+
}
65+
66+
public void setReason(String reason) {
67+
this.reason = reason;
68+
}
69+
}
70+
71+
class HostUpdated {
72+
private Host host;
73+
private Date start;
74+
private Date end;
75+
private String outputMsg;
76+
77+
public HostUpdated(Host host, Date start, Date end, String outputMsg) {
78+
this.host = host;
79+
this.start = start;
80+
this.end = end;
81+
this.outputMsg = outputMsg;
82+
}
83+
84+
public Host getHost() {
85+
return host;
86+
}
87+
88+
public void setHost(Host host) {
89+
this.host = host;
90+
}
91+
92+
public Date getStart() {
93+
return start;
94+
}
95+
96+
public void setStart(Date start) {
97+
this.start = start;
98+
}
99+
100+
public Date getEnd() {
101+
return end;
102+
}
103+
104+
public void setEnd(Date end) {
105+
this.end = end;
106+
}
107+
108+
public String getOutputMsg() {
109+
return outputMsg;
110+
}
111+
112+
public void setOutputMsg(String outputMsg) {
113+
this.outputMsg = outputMsg;
114+
}
115+
}
116+
117+
enum Stage {
118+
PreFlight, PreMaintenance, Maintenance, PostMaintenance;
119+
120+
public Stage next() {
121+
switch (this) {
122+
case PreFlight:
123+
return PreMaintenance;
124+
case PreMaintenance:
125+
return Maintenance;
126+
case Maintenance:
127+
return PostMaintenance;
128+
case PostMaintenance:
129+
return null;
130+
}
131+
throw new CloudRuntimeException("Unexpected stage: " + this);
132+
}
133+
}
134+
135+
enum ResourceType {
136+
Pod, Cluster, Zone, Host
137+
}
138+
139+
/**
140+
* Starts rolling maintenance as specified in cmd
141+
* @param cmd command
142+
* @return tuple: (SUCCESS, DETAILS, (HOSTS_UPDATED, HOSTS_SKIPPED))
143+
*/
144+
Ternary<Boolean, String, Pair<List<HostUpdated>, List<HostSkipped>>> startRollingMaintenance(StartRollingMaintenanceCmd cmd);
145+
Pair<ResourceType, List<Long>> getResourceTypeIdPair(StartRollingMaintenanceCmd cmd);
146+
}

‎api/src/main/java/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ public class ApiConstants {
7474
public static final String CLEANUP = "cleanup";
7575
public static final String MAKEREDUNDANT = "makeredundant";
7676
public static final String CLUSTER_ID = "clusterid";
77+
public static final String CLUSTER_IDS = "clusterids";
7778
public static final String CLUSTER_NAME = "clustername";
7879
public static final String CLUSTER_TYPE = "clustertype";
7980
public static final String CN = "cn";
@@ -173,6 +174,7 @@ public class ApiConstants {
173174
public static final String HEALTH = "health";
174175
public static final String HIDE_IP_ADDRESS_USAGE = "hideipaddressusage";
175176
public static final String HOST_ID = "hostid";
177+
public static final String HOST_IDS = "hostids";
176178
public static final String HOST_NAME = "hostname";
177179
public static final String HYPERVISOR = "hypervisor";
178180
public static final String INLINE = "inline";
@@ -256,6 +258,7 @@ public class ApiConstants {
256258
public static final String OS_NAME_FOR_HYPERVISOR = "osnameforhypervisor";
257259
public static final String OUTOFBANDMANAGEMENT_POWERSTATE = "outofbandmanagementpowerstate";
258260
public static final String OUTOFBANDMANAGEMENT_ENABLED = "outofbandmanagementenabled";
261+
public static final String OUTPUT = "output";
259262
public static final String OVF_PROPERTIES = "ovfproperties";
260263
public static final String PARAMS = "params";
261264
public static final String PARENT_ID = "parentid";
@@ -267,6 +270,7 @@ public class ApiConstants {
267270
public static final String PASSWORD_ENABLED = "passwordenabled";
268271
public static final String SSHKEY_ENABLED = "sshkeyenabled";
269272
public static final String PATH = "path";
273+
public static final String PAYLOAD = "payload";
270274
public static final String POD_ID = "podid";
271275
public static final String POD_NAME = "podname";
272276
public static final String POD_IDS = "podids";

‎api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
import java.util.Map;
2323
import java.util.Set;
2424

25+
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
26+
import com.cloud.resource.RollingMaintenanceManager;
27+
import org.apache.cloudstack.api.response.RollingMaintenanceResponse;
28+
import org.apache.cloudstack.management.ManagementServerHost;
2529
import org.apache.cloudstack.affinity.AffinityGroup;
2630
import org.apache.cloudstack.affinity.AffinityGroupResponse;
2731
import org.apache.cloudstack.api.ApiConstants.HostDetails;
@@ -88,7 +92,6 @@
8892
import org.apache.cloudstack.api.response.ResourceCountResponse;
8993
import org.apache.cloudstack.api.response.ResourceLimitResponse;
9094
import org.apache.cloudstack.api.response.ResourceTagResponse;
91-
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
9295
import org.apache.cloudstack.api.response.SSHKeyPairResponse;
9396
import org.apache.cloudstack.api.response.SecurityGroupResponse;
9497
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
@@ -125,7 +128,6 @@
125128
import org.apache.cloudstack.backup.Backup;
126129
import org.apache.cloudstack.backup.BackupSchedule;
127130
import org.apache.cloudstack.config.Configuration;
128-
import org.apache.cloudstack.management.ManagementServerHost;
129131
import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule;
130132
import org.apache.cloudstack.region.PortableIp;
131133
import org.apache.cloudstack.region.PortableIpRange;
@@ -482,4 +484,7 @@ List<TemplateResponse> createTemplateResponses(ResponseView view, VirtualMachine
482484
ManagementServerResponse createManagementResponse(ManagementServerHost mgmt);
483485

484486
List<RouterHealthCheckResultResponse> createHealthCheckResponse(VirtualMachine router, List<RouterHealthCheckResult> healthCheckResults);
487+
488+
RollingMaintenanceResponse createRollingMaintenanceResponse(Boolean success, String details, List<RollingMaintenanceManager.HostUpdated> hostsUpdated, List<RollingMaintenanceManager.HostSkipped> hostsSkipped);
489+
485490
}

0 commit comments

Comments
 (0)