Commit d28a9193 authored by Maxime COMPASTIÉ's avatar Maxime COMPASTIÉ
Browse files

Openstack cloud

parent 9f053c06
......@@ -157,9 +157,6 @@ public class PAGateway {
em.getTransaction().begin();
//TODO: Add docker job handling
//TODO: Add spark job handling (throw not handled yet exception)
Job newJob = new Job();
newJob.setJobId(job.optJSONObject("jobInformation").optString("id"));
newJob.setName(job.optJSONObject("jobInformation").optString("name"));
......@@ -242,24 +239,47 @@ public class PAGateway {
* @param deployment The deployment information object
*/
private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, Deployment deployment) {
File fXmlFile = null;
try {
fXmlFile = new File(getClass().getResource("/Define_NS_AWS.xml").toURI());
} catch (URISyntaxException e) {
LOGGER.error(e.getStackTrace());
}
String filename;
Map<String, String> variables = new HashMap<>();
variables.put("NS_name", nodeSourceName);
variables.put("NS_nVMs", "0");
variables.put("aws_username", cloud.getCredentials().getUserName());
variables.put("aws_secret", cloud.getCredentials().getPrivateKey());
variables.put("security_group", cloud.getSecurityGroup());
variables.put("image", deployment.getLocationName() + "/" + deployment.getImageProviderId());
try {
variables.put("rm_host_name", (new URL(this.paURL)).getHost());
URL endpointPa = (new URL(this.paURL));
variables.put("rm_host_name", endpointPa.getHost());
variables.put("pa_port", "" + endpointPa.getPort());
} catch (MalformedURLException e) {
LOGGER.error(e.getStackTrace());
}
switch (cloud.getCloudProviderName()) {
case "aws-ec2":
filename = "/Define_NS_AWS.xml";
variables.put("aws_username", cloud.getCredentials().getUserName());
variables.put("aws_secret", cloud.getCredentials().getPrivateKey());
break;
case "openstack":
filename = "/Define_NS_OS.xml";
variables.put("os_endpoint", cloud.getEndpoint());
variables.put("os_scopePrefix", cloud.getScopePrefix());
variables.put("os_scopeValue", cloud.getScopeValue());
variables.put("os_identityVersion", cloud.getIdentityVersion());
variables.put("os_username", cloud.getCredentials().getUserName());
variables.put("os_password", cloud.getCredentials().getPrivateKey());
variables.put("os_domain", cloud.getCredentials().getDomain());
variables.put("os_region", deployment.getLocationName());
variables.put("os_networkId",cloud.getDefaultNetwork());
break;
default:
throw new IllegalArgumentException("Spark tasks are not handled yet.");
}
File fXmlFile = null;
try {
fXmlFile = new File(getClass().getResource(filename).toURI());
} catch (URISyntaxException e) {
LOGGER.error(e.getStackTrace());
}
assert fXmlFile != null;
LOGGER.info("Submitting the file: " + fXmlFile.toString());
LOGGER.info("Trying to deploy the NS: " + nodeSourceName);
schedulerGateway.submit(fXmlFile, variables);
......@@ -439,15 +459,22 @@ public class PAGateway {
} else {
newCloud.setSecurityGroup(cloud.optString("securityGroup"));
}
newCloud.setEndpoint(cloud.optString("endpoint"));
newCloud.setScopePrefix(cloud.optJSONObject("scope").optString("prefix"));
newCloud.setScopeValue(cloud.optJSONObject("scope").optString("value"));
newCloud.setIdentityVersion(cloud.optString("identityVersion"));
newCloud.setDefaultNetwork(cloud.optString("defaultNetwork"));
newCloud.setBlacklist(cloud.optString("blacklist"));
Credentials credentials = new Credentials();
credentials.setUserName(cloud.optJSONObject("credentials").optString("user"));
credentials.setPrivateKey(cloud.optJSONObject("credentials").optString("secret"));
credentials.setDomain(cloud.optJSONObject("credentials").optString("domain"));
em.persist(credentials);
newCloud.setCredentials(credentials);
String dummyInfraName = "iamadummy" + newCloud.getCloudProviderName();
connectorIaasGateway.defineInfrastructure(dummyInfraName, newCloud.getCloudProviderName(), credentials, "");
connectorIaasGateway.defineInfrastructure(dummyInfraName, newCloud, "");
newCloud.setDummyInfrastructureName(dummyInfraName);
em.persist(newCloud);
......
......@@ -2,6 +2,7 @@ package org.activeeon.morphemic.infrastructure.deployment;
import lombok.SneakyThrows;
import org.activeeon.morphemic.model.Credentials;
import org.activeeon.morphemic.model.PACloud;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.Validate;
import org.apache.http.client.utils.URIBuilder;
......@@ -128,9 +129,9 @@ public class PAConnectorIaasGateway {
}
@SneakyThrows
public void defineInfrastructure(String infrastructureName, String cloudProviderName, Credentials credentials, String region) {
public void defineInfrastructure(String infrastructureName, PACloud cloud, String region) {
Validate.notNull(infrastructureName, "infrastructureName must not be null");
Validate.notNull(cloudProviderName, "cloudProviderName must not be null");
Validate.notNull(cloud.getCloudProviderName(), "cloudProviderName must not be null");
URIBuilder uriBuilder = new URIBuilder(new URL(paURL).toURI());
URI requestUri = uriBuilder.setPath(CONNECTOR_IAAS_PATH + "/infrastructures").build();
......@@ -140,9 +141,26 @@ public class PAConnectorIaasGateway {
connection.setRequestProperty("Content-Type", "application/json; utf-8");
connection.setDoOutput(true);
String jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," +
"\"type\": \"" + cloudProviderName + "\"," +
"\"credentials\": {\"username\": \"" + credentials.getUserName() + "\", \"password\": \"" + credentials.getPrivateKey() + "\"}, \"region\": \"" + region + "\"}";
String jsonOutputString;
switch (cloud.getCloudProviderName()) {
case "aws-ec2":
jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," +
"\"type\": \"" + cloud.getCloudProviderName() + "\"," +
"\"credentials\": {\"username\": \"" + cloud.getCredentials().getUserName() + "\", \"password\": \"" +
cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}";
break;
case "openstack":
jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," +
"\"type\": \"openstack-nova\", \"endpoint\": \"" + cloud.getEndpoint() +
"\", \"scope\":{\"prefix\": \"" + cloud.getScopePrefix() + "\", \"value\":\"" +
cloud.getScopeValue() + "\"}, \"identityVersion\": \"" + cloud.getIdentityVersion() + "\", " +
"\"credentials\": {\"username\": \"" + cloud.getCredentials().getUserName() +
"\", \"password\": \"" + cloud.getCredentials().getPrivateKey() + "\", \"domain\": \"" +
cloud.getCredentials().getDomain() + "\"}, \"region\": \"" + region + "\"}";
break;
default:
throw new IllegalArgumentException("The infrastructure " + cloud.getCloudProviderName() + " is not handled yet.");
}
try(OutputStream os = connection.getOutputStream()) {
byte[] input = jsonOutputString.getBytes("utf-8");
......
......@@ -29,4 +29,7 @@ public class Credentials implements Serializable {
@Column(name = "PUBLIC_KEY")
private String publicKey;
@Column(name = "DOMAIN")
private String domain;
}
......@@ -36,9 +36,27 @@ public class PACloud implements Serializable {
@Column(name = "SECURITY_GROUP")
private String securityGroup;
@Column(name = "ENDPOINT")
private String endpoint;
@Column(name = "SCOPE_PREFIX")
private String scopePrefix;
@Column(name = "SCOPE_VALUE")
private String scopeValue;
@Column(name = "IDENTITY_VERSION")
private String identityVersion;
@Column(name = "DUMMY_INFRASTRUCTURE_NAME")
private String dummyInfrastructureName;
@Column(name = "DEFAULT_NETWORK")
private String defaultNetwork;
@Column(name = "BLACKLIST")
private String blacklist;
@Column(name = "DEPLOYED_REGIONS")
@ElementCollection(targetClass=String.class)
private Map<String, String> deployedRegions;
......
......@@ -52,6 +52,6 @@ public class GeoLocationUtils {
return cloudsGeoLocationData.stream()
.filter(cloudGL -> cloud.equals(cloudGL.getCloud()) && region.equals(cloudGL.getRegion()))
.findAny()
.orElse(null);
.orElse(new GeoLocationData());
}
}
......@@ -11,6 +11,8 @@ import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;
import java.math.BigDecimal;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class NodeCandidateUtils {
......@@ -115,7 +117,7 @@ public class NodeCandidateUtils {
hardware.setId(hardwareId);
hardware.setName(hardwareJSON.optString("type"));
hardware.setProviderId(paCloud.getCloudID());
hardware.setCores(Integer.valueOf(hardwareJSON.optString("minCores")));
hardware.setCores(Math.round(Float.parseFloat(hardwareJSON.optString("minCores"))));
hardware.setRam(Long.valueOf(hardwareJSON.optString("minRam")));
if ("aws-ec2".equals(nodeCandidateJSON.optString("cloud"))) {
hardware.setDisk((double) 8);
......@@ -224,16 +226,28 @@ public class NodeCandidateUtils {
newCloudIds.forEach(newCloudId -> {
PACloud paCloud = em.find(PACloud.class, newCloudId);
List<String> blacklistedRegions = Arrays.asList(paCloud.getBlacklist().split(","));
JSONArray images = connectorIaasGateway.getImages(paCloud.getDummyInfrastructureName());
List<JSONObject> consolidatedImages = images.toList().parallelStream().map(o -> (JSONObject) o).filter(record -> !blacklistedRegions.contains(record.get("location"))).collect(Collectors.toList());
//TODO: (Optimization) An images per region map structure <region,[image1,image2]> could be the best here.
// It can reduce the getNodeCandidates calls to PA.
images.forEach(img -> {
JSONObject image = (JSONObject) img;
consolidatedImages.forEach(image -> {
String region = image.optString("location");
String imageReq;
switch (paCloud.getCloudProviderName()) {
case "aws-ec2":
imageReq = "Linux";
break;
case "openstack":
imageReq = "linux";
break;
default:
throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudProviderName() + " is not handled yet.");
}
JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(),
region, "Linux");
region, imageReq);
nodeCandidates.forEach(nc -> {
JSONObject nodeCandidate = (JSONObject) nc;
em.persist(createLocation(nodeCandidate, paCloud));
......
<?xml version="1.0" encoding="UTF-8"?>
<job
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="urn:proactive:jobdescriptor:3.12" xsi:schemaLocation="urn:proactive:jobdescriptor:3.12 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.12/schedulerjob.xsd" name="Define_NS_OS" projectName="Morphemic" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" >
<variables>
<variable name="NS_name" value="" />
<variable name="NS_nVMs" value="0" model="PA:Integer"/>
<variable name="os_username" value="" model="PA:HIDDEN"/>
<variable name="os_password" value="" model="PA:HIDDEN"/>
<variable name="security_group" value="" />
<variable name="rm_host_name" value="" />
<variable name="os_domain" value="" />
<variable name="os_endpoint" value="" />
<variable name="os_scopePrefix" value="" />
<variable name="os_scopeValue" value="" />
<variable name="os_region" value="" />
<variable name="os_identityVersion" value="" />
<variable name="os_networkId" value="" />
<variable name="pa_port" value="" />
</variables>
<description>
<![CDATA[ A workflow that executes Groovy in JVM. ]]>
</description>
<taskFlow>
<task name="defineOpenStackNodeSource"
fork="true">
<description>
<![CDATA[ Define an OpenStack NodeSource ]]>
</description>
<scriptExecutable>
<script>
<code language="groovy">
<![CDATA[
// Conncting to the Scheduler
def retCode = 0;
def nodeSourceName = variables.get("NS_name")
def protocol = (variables.get("pa_port") == "443" || variables.get("pa_port") == "8443" ) ? "https" : "http"
println "[+] Preparation of NodeSoure " + nodeSourceName
print "(1/4) Connecting to the RM ..."
rmapi.connect()
println " ... OK !"
// Getting NS configuration settings
def infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.OpenstackInfrastructure"
def infrastructureParameters = [variables.get("os_username"), //username
variables.get("os_password"), //password
variables.get("os_domain"), // domain
variables.get("os_endpoint"), // endpoint
variables.get("os_scopePrefix"), // scopePrefix
variables.get("os_scopeValue"), // scopeValue
variables.get("os_region"), // region
variables.get("os_identityVersion"), // identityVerion
"irrelevant_image", //image
"irrelevant_flavor", //flavor
variables.get("os_networkId"), // networkId
"", //publicKeyName
variables.get("NS_nVMs"), //numberOfInstance
"1", //N VMs per node
protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/connector-iaas", //connector-iaas url
variables.get("rm_host_name"), //rmHostname
protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", //node jar url
"-Dproactive.useIPaddress=true", //additionalProperties"
"300000", //nodesInitDelay
""] //startupScript
def infrastructureFileParameters = []
def policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.EmptyPolicy"
def poliyParameters = ["ALL","ME"]
def policyFileParameters = []
def nodesRecoverable = "true"
// Enforcing ....
try {
print "(2/4) Creating NodeSource ..."
print rmapi.defineNodeSource(nodeSourceName,infrastructureType,(String[]) infrastructureParameters.toArray(),(String[]) infrastructureFileParameters.toArray(), policyType, (String[]) poliyParameters.toArray(), (String[]) policyFileParameters.toArray(),nodesRecoverable)
println " ... OK !"
print "(3/4) Deploying the NodeSource ..."
rmapi.deployNodeSource(nodeSourceName)
println " ... OK !"
} catch (Exception e) {
println " ... Error: " + e.getMessage();
retCode = 1;
} finally {
print "(4/4) Logging out ..."
rmapi.disconnect();
println " ... OK !"
}
return retCode;
]]>
</code>
</script>
</scriptExecutable>
<metadata>
<positionTop>
142.13333129882812
</positionTop>
<positionLeft>
351.5
</positionLeft>
</metadata>
</task>
</taskFlow>
<metadata>
<visualization>
<![CDATA[ <html>
<head>
<link rel="stylesheet" href="/studio/styles/studio-standalone.css">
<style>
#workflow-designer {
left:0 !important;
top:0 !important;
width:2826px;
height:7788px;
}
</style>
</head>
<body>
<div id="workflow-visualization-view"><div id="workflow-visualization" style="position:relative;top:-137.13333129882812px;left:-346.5px"><div class="task _jsPlumb_endpoint_anchor_ ui-draggable active-task" style="top: 142.134px; left: 351.5px;" id="jsPlumb_1_22"><a class="task-name" data-toggle="tooltip" data-placement="right" title="Define an OpenStack NodeSource"><img src="images/Groovy.png" width="20px">&nbsp;<span class="name">defineOpenStackNodeSource</span></a></div><div style="position: absolute; height: 20px; width: 20px; left: 425.5px; top: 172px;" class="_jsPlumb_endpoint source-endpoint dependency-source-endpoint connected _jsPlumb_endpoint_anchor_ ui-draggable ui-droppable"><svg style="position:absolute;left:0px;top:0px" width="20" height="20" pointer-events="all" position="absolute" version="1.1" xmlns="http://www.w3.org/1999/xhtml"><circle cx="10" cy="10" r="10" version="1.1" xmlns="http://www.w3.org/1999/xhtml" fill="#666" stroke="none" style=""></circle></svg></div></div></div>
</body>
</html>
]]>
</visualization>
</metadata>
</job>
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment