From a896ddf995d9d62d2766028c5482a1a4581ccb83 Mon Sep 17 00:00:00 2001 From: mklkun Date: Thu, 17 Dec 2020 15:44:01 +0100 Subject: [PATCH 1/7] Handle Openstack in addCloud and compute NodeCandidates --- .../org/activeeon/morphemic/PAGateway.java | 7 ++++- .../deployment/PAConnectorIaasGateway.java | 28 +++++++++++++++---- .../morphemic/model/Credentials.java | 3 ++ .../activeeon/morphemic/model/PACloud.java | 12 ++++++++ .../morphemic/service/GeoLocationUtils.java | 2 +- .../morphemic/service/NodeCandidateUtils.java | 15 ++++++++-- 6 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java index 80056a45..b367407f 100644 --- a/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -439,15 +439,20 @@ public class PAGateway { } else { newCloud.setSecurityGroup(cloud.optString("securityGroup")); } + newCloud.setEndpoint(cloud.optString("endpoint")); + newCloud.setScopePrefix(cloud.optJSONObject("scope").optString("prefix")); + newCloud.setScopeValue(cloud.optJSONObject("scope").optString("value")); + newCloud.setIdentityVersion(cloud.optString("identityVersion")); Credentials credentials = new Credentials(); credentials.setUserName(cloud.optJSONObject("credentials").optString("user")); credentials.setPrivateKey(cloud.optJSONObject("credentials").optString("secret")); + credentials.setDomain(cloud.optJSONObject("credentials").optString("domain")); em.persist(credentials); newCloud.setCredentials(credentials); String dummyInfraName = "iamadummy" + newCloud.getCloudProviderName(); - connectorIaasGateway.defineInfrastructure(dummyInfraName, newCloud.getCloudProviderName(), credentials, ""); + connectorIaasGateway.defineInfrastructure(dummyInfraName, newCloud, ""); newCloud.setDummyInfrastructureName(dummyInfraName); em.persist(newCloud); diff --git a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java index 2d1dce9a..db93ef5d 100644 --- a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java +++ b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java @@ -2,6 +2,7 @@ package org.activeeon.morphemic.infrastructure.deployment; import lombok.SneakyThrows; import org.activeeon.morphemic.model.Credentials; +import org.activeeon.morphemic.model.PACloud; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.Validate; import org.apache.http.client.utils.URIBuilder; @@ -128,9 +129,9 @@ public class PAConnectorIaasGateway { } @SneakyThrows - public void defineInfrastructure(String infrastructureName, String cloudProviderName, Credentials credentials, String region) { + public void defineInfrastructure(String infrastructureName, PACloud cloud, String region) { Validate.notNull(infrastructureName, "infrastructureName must not be null"); - Validate.notNull(cloudProviderName, "cloudProviderName must not be null"); + Validate.notNull(cloud.getCloudProviderName(), "cloudProviderName must not be null"); URIBuilder uriBuilder = new URIBuilder(new URL(paURL).toURI()); URI requestUri = uriBuilder.setPath(CONNECTOR_IAAS_PATH + "/infrastructures").build(); @@ -140,9 +141,26 @@ public class PAConnectorIaasGateway { connection.setRequestProperty("Content-Type", "application/json; utf-8"); connection.setDoOutput(true); - String jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + - "\"type\": \"" + cloudProviderName + "\"," + - "\"credentials\": {\"username\": \"" + credentials.getUserName() + "\", \"password\": \"" + credentials.getPrivateKey() + "\"}, \"region\": \"" + region + "\"}"; + String jsonOutputString; + switch (cloud.getCloudProviderName()) { + case "aws-ec2": + jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + + "\"type\": \"" + cloud.getCloudProviderName() + "\"," + + "\"credentials\": {\"username\": \"" + cloud.getCredentials().getUserName() + "\", \"password\": \"" + + cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}"; + break; + case "openstack": + jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + + "\"type\": \"openstack-nova\", \"endpoint\": \"" + cloud.getEndpoint() + + "\", \"scope\":{\"prefix\": \"" + cloud.getScopePrefix() + "\", \"value\":\"" + + cloud.getScopeValue() + "\"}, \"identityVersion\": \"" + cloud.getIdentityVersion() + "\", " + + "\"credentials\": {\"username\": \"" + cloud.getCredentials().getUserName() + + "\", \"password\": \"" + cloud.getCredentials().getPrivateKey() + "\", \"domain\": \"" + + cloud.getCredentials().getDomain() + "\"}, \"region\": \"" + region + "\"}"; + break; + default: + throw new IllegalArgumentException("The infrastructure " + cloud.getCloudProviderName() + " is not handled yet."); + } try(OutputStream os = connection.getOutputStream()) { byte[] input = jsonOutputString.getBytes("utf-8"); diff --git a/src/main/java/org/activeeon/morphemic/model/Credentials.java b/src/main/java/org/activeeon/morphemic/model/Credentials.java index 2ca1d4d5..a081ac4d 100644 --- a/src/main/java/org/activeeon/morphemic/model/Credentials.java +++ b/src/main/java/org/activeeon/morphemic/model/Credentials.java @@ -29,4 +29,7 @@ public class Credentials implements Serializable { @Column(name = "PUBLIC_KEY") private String publicKey; + + @Column(name = "DOMAIN") + private String domain; } diff --git a/src/main/java/org/activeeon/morphemic/model/PACloud.java b/src/main/java/org/activeeon/morphemic/model/PACloud.java index e9824722..8ad3a271 100644 --- a/src/main/java/org/activeeon/morphemic/model/PACloud.java +++ b/src/main/java/org/activeeon/morphemic/model/PACloud.java @@ -36,6 +36,18 @@ public class PACloud implements Serializable { @Column(name = "SECURITY_GROUP") private String securityGroup; + @Column(name = "ENDPOINT") + private String endpoint; + + @Column(name = "SCOPE_PREFIX") + private String scopePrefix; + + @Column(name = "SCOPE_VALUE") + private String scopeValue; + + @Column(name = "IDENTITY_VERSION") + private String identityVersion; + @Column(name = "DUMMY_INFRASTRUCTURE_NAME") private String dummyInfrastructureName; diff --git a/src/main/java/org/activeeon/morphemic/service/GeoLocationUtils.java b/src/main/java/org/activeeon/morphemic/service/GeoLocationUtils.java index dd75248b..36712f9d 100644 --- a/src/main/java/org/activeeon/morphemic/service/GeoLocationUtils.java +++ b/src/main/java/org/activeeon/morphemic/service/GeoLocationUtils.java @@ -52,6 +52,6 @@ public class GeoLocationUtils { return cloudsGeoLocationData.stream() .filter(cloudGL -> cloud.equals(cloudGL.getCloud()) && region.equals(cloudGL.getRegion())) .findAny() - .orElse(null); + .orElse(new GeoLocationData()); } } diff --git a/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java b/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java index af79dba6..c8c03562 100644 --- a/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java +++ b/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java @@ -115,7 +115,7 @@ public class NodeCandidateUtils { hardware.setId(hardwareId); hardware.setName(hardwareJSON.optString("type")); hardware.setProviderId(paCloud.getCloudID()); - hardware.setCores(Integer.valueOf(hardwareJSON.optString("minCores"))); + hardware.setCores(Math.round(Float.parseFloat(hardwareJSON.optString("minCores")))); hardware.setRam(Long.valueOf(hardwareJSON.optString("minRam"))); if ("aws-ec2".equals(nodeCandidateJSON.optString("cloud"))) { hardware.setDisk((double) 8); @@ -232,8 +232,19 @@ public class NodeCandidateUtils { images.forEach(img -> { JSONObject image = (JSONObject) img; String region = image.optString("location"); + String imageReq; + switch (paCloud.getCloudProviderName()) { + case "aws-ec2": + imageReq = "Linux"; + break; + case "openstack": + imageReq = "linux"; + break; + default: + throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudProviderName() + " is not handled yet."); + } JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(), - region, "Linux"); + region, imageReq); nodeCandidates.forEach(nc -> { JSONObject nodeCandidate = (JSONObject) nc; em.persist(createLocation(nodeCandidate, paCloud)); -- GitLab From 45c3f21c2fc8f7aad1b3b345acf30c42c5967af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20COMPASTI=C3=89?= Date: Tue, 22 Dec 2020 02:12:49 +0100 Subject: [PATCH 2/7] OpenStack addition WF (tested against UiO infra) --- src/main/resources/Define_NS_OS.xml | 127 ++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/main/resources/Define_NS_OS.xml diff --git a/src/main/resources/Define_NS_OS.xml b/src/main/resources/Define_NS_OS.xml new file mode 100644 index 00000000..072fc5d8 --- /dev/null +++ b/src/main/resources/Define_NS_OS.xml @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 142.13333129882812 + + + 351.5 + + + + + + + + + + + + + + + + ]]> + + + \ No newline at end of file -- GitLab From c736c576f15b3a9bbca2d36db06f918b9ba05c84 Mon Sep 17 00:00:00 2001 From: mklkun Date: Tue, 22 Dec 2020 16:41:55 +0300 Subject: [PATCH 3/7] Handle Nodesource creation when adding a deployment --- .../org/activeeon/morphemic/PAGateway.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java index b367407f..4c7cc6b0 100644 --- a/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -157,9 +157,6 @@ public class PAGateway { em.getTransaction().begin(); - //TODO: Add docker job handling - //TODO: Add spark job handling (throw not handled yet exception) - Job newJob = new Job(); newJob.setJobId(job.optJSONObject("jobInformation").optString("id")); newJob.setName(job.optJSONObject("jobInformation").optString("name")); @@ -242,17 +239,11 @@ public class PAGateway { * @param deployment The deployment information object */ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, Deployment deployment) { - File fXmlFile = null; - try { - fXmlFile = new File(getClass().getResource("/Define_NS_AWS.xml").toURI()); - } catch (URISyntaxException e) { - LOGGER.error(e.getStackTrace()); - } + //TODO: variables and filename to adapt according to the created WF input variables/file + String filename; Map variables = new HashMap<>(); variables.put("NS_name", nodeSourceName); variables.put("NS_nVMs", "0"); - variables.put("aws_username", cloud.getCredentials().getUserName()); - variables.put("aws_secret", cloud.getCredentials().getPrivateKey()); variables.put("security_group", cloud.getSecurityGroup()); variables.put("image", deployment.getLocationName() + "/" + deployment.getImageProviderId()); try { @@ -260,6 +251,32 @@ public class PAGateway { } catch (MalformedURLException e) { LOGGER.error(e.getStackTrace()); } + switch (cloud.getCloudProviderName()) { + case "aws-ec2": + filename = "/Define_NS_AWS.xml"; + variables.put("aws_username", cloud.getCredentials().getUserName()); + variables.put("aws_secret", cloud.getCredentials().getPrivateKey()); + break; + case "openstack": + filename = "/Define_NS_OS.xml"; + variables.put("endpoint", cloud.getEndpoint()); + variables.put("scope_prefix", cloud.getScopePrefix()); + variables.put("scope_value", cloud.getScopeValue()); + variables.put("identityVersion", cloud.getIdentityVersion()); + variables.put("os_username", cloud.getCredentials().getUserName()); + variables.put("os_secret", cloud.getCredentials().getPrivateKey()); + variables.put("os_domain", cloud.getCredentials().getDomain()); + break; + default: + throw new IllegalArgumentException("Spark tasks are not handled yet."); + } + File fXmlFile = null; + try { + fXmlFile = new File(getClass().getResource(filename).toURI()); + } catch (URISyntaxException e) { + LOGGER.error(e.getStackTrace()); + } + assert fXmlFile != null; LOGGER.info("Submitting the file: " + fXmlFile.toString()); LOGGER.info("Trying to deploy the NS: " + nodeSourceName); schedulerGateway.submit(fXmlFile, variables); -- GitLab From b4a57a967a4b9d807e43684600bbb78b5ebfef6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20COMPASTI=C3=89?= Date: Tue, 22 Dec 2020 18:33:00 +0100 Subject: [PATCH 4/7] Integrating OS deployment + networkId & defaultNetwork parameter --- .../java/org/activeeon/morphemic/PAGateway.java | 17 +++++++++++------ .../org/activeeon/morphemic/model/PACloud.java | 3 +++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java index 4c7cc6b0..64204b09 100644 --- a/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -247,7 +247,9 @@ public class PAGateway { variables.put("security_group", cloud.getSecurityGroup()); variables.put("image", deployment.getLocationName() + "/" + deployment.getImageProviderId()); try { - variables.put("rm_host_name", (new URL(this.paURL)).getHost()); + URL endpointPa = (new URL(this.paURL)); + variables.put("rm_host_name", endpointPa.getHost()); + variables.put("pa_port", "" + endpointPa.getPort()); } catch (MalformedURLException e) { LOGGER.error(e.getStackTrace()); } @@ -259,13 +261,15 @@ public class PAGateway { break; case "openstack": filename = "/Define_NS_OS.xml"; - variables.put("endpoint", cloud.getEndpoint()); - variables.put("scope_prefix", cloud.getScopePrefix()); - variables.put("scope_value", cloud.getScopeValue()); - variables.put("identityVersion", cloud.getIdentityVersion()); + variables.put("os_endpoint", cloud.getEndpoint()); + variables.put("os_scopePrefix", cloud.getScopePrefix()); + variables.put("os_scopeValue", cloud.getScopeValue()); + variables.put("os_identityVersion", cloud.getIdentityVersion()); variables.put("os_username", cloud.getCredentials().getUserName()); - variables.put("os_secret", cloud.getCredentials().getPrivateKey()); + variables.put("os_password", cloud.getCredentials().getPrivateKey()); variables.put("os_domain", cloud.getCredentials().getDomain()); + variables.put("os_region", deployment.getLocationName()); + variables.put("os_networkId",cloud.getDefaultNetwork()); break; default: throw new IllegalArgumentException("Spark tasks are not handled yet."); @@ -460,6 +464,7 @@ public class PAGateway { newCloud.setScopePrefix(cloud.optJSONObject("scope").optString("prefix")); newCloud.setScopeValue(cloud.optJSONObject("scope").optString("value")); newCloud.setIdentityVersion(cloud.optString("identityVersion")); + newCloud.setDefaultNetwork(cloud.optString("defaultNetwork")); Credentials credentials = new Credentials(); credentials.setUserName(cloud.optJSONObject("credentials").optString("user")); diff --git a/src/main/java/org/activeeon/morphemic/model/PACloud.java b/src/main/java/org/activeeon/morphemic/model/PACloud.java index 8ad3a271..5c5f4d45 100644 --- a/src/main/java/org/activeeon/morphemic/model/PACloud.java +++ b/src/main/java/org/activeeon/morphemic/model/PACloud.java @@ -51,6 +51,9 @@ public class PACloud implements Serializable { @Column(name = "DUMMY_INFRASTRUCTURE_NAME") private String dummyInfrastructureName; + @Column(name = "defaultNetwork") + private String defaultNetwork; + @Column(name = "DEPLOYED_REGIONS") @ElementCollection(targetClass=String.class) private Map deployedRegions; -- GitLab From 9952ee534a8fb826dfb379f5532ff419ca6b34ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20COMPASTI=C3=89?= Date: Tue, 22 Dec 2020 18:37:13 +0100 Subject: [PATCH 5/7] Minor fixes --- src/main/java/org/activeeon/morphemic/PAGateway.java | 1 - src/main/java/org/activeeon/morphemic/model/PACloud.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java index 64204b09..d0eee99b 100644 --- a/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -239,7 +239,6 @@ public class PAGateway { * @param deployment The deployment information object */ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, Deployment deployment) { - //TODO: variables and filename to adapt according to the created WF input variables/file String filename; Map variables = new HashMap<>(); variables.put("NS_name", nodeSourceName); diff --git a/src/main/java/org/activeeon/morphemic/model/PACloud.java b/src/main/java/org/activeeon/morphemic/model/PACloud.java index 5c5f4d45..be36dd9e 100644 --- a/src/main/java/org/activeeon/morphemic/model/PACloud.java +++ b/src/main/java/org/activeeon/morphemic/model/PACloud.java @@ -51,7 +51,7 @@ public class PACloud implements Serializable { @Column(name = "DUMMY_INFRASTRUCTURE_NAME") private String dummyInfrastructureName; - @Column(name = "defaultNetwork") + @Column(name = "DEFAULTNETWORK") private String defaultNetwork; @Column(name = "DEPLOYED_REGIONS") -- GitLab From 7c5833ca3d9a9bd6c9c49620215b6a89ac62aa19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20COMPASTI=C3=89?= Date: Tue, 22 Dec 2020 19:06:55 +0100 Subject: [PATCH 6/7] Support for region blacklist --- src/main/java/org/activeeon/morphemic/PAGateway.java | 1 + src/main/java/org/activeeon/morphemic/model/PACloud.java | 3 +++ .../activeeon/morphemic/service/NodeCandidateUtils.java | 7 +++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java index d0eee99b..0fbf7b5b 100644 --- a/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -464,6 +464,7 @@ public class PAGateway { newCloud.setScopeValue(cloud.optJSONObject("scope").optString("value")); newCloud.setIdentityVersion(cloud.optString("identityVersion")); newCloud.setDefaultNetwork(cloud.optString("defaultNetwork")); + newCloud.setBlacklist(cloud.optString("blacklist")); Credentials credentials = new Credentials(); credentials.setUserName(cloud.optJSONObject("credentials").optString("user")); diff --git a/src/main/java/org/activeeon/morphemic/model/PACloud.java b/src/main/java/org/activeeon/morphemic/model/PACloud.java index be36dd9e..8de7793f 100644 --- a/src/main/java/org/activeeon/morphemic/model/PACloud.java +++ b/src/main/java/org/activeeon/morphemic/model/PACloud.java @@ -54,6 +54,9 @@ public class PACloud implements Serializable { @Column(name = "DEFAULTNETWORK") private String defaultNetwork; + @Column(name = "BLACKLIST") + private String blacklist; + @Column(name = "DEPLOYED_REGIONS") @ElementCollection(targetClass=String.class) private Map deployedRegions; diff --git a/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java b/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java index c8c03562..9feb5d6f 100644 --- a/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java +++ b/src/main/java/org/activeeon/morphemic/service/NodeCandidateUtils.java @@ -11,6 +11,8 @@ import javax.persistence.EntityManagerFactory; import javax.persistence.Persistence; import java.math.BigDecimal; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class NodeCandidateUtils { @@ -224,13 +226,14 @@ public class NodeCandidateUtils { newCloudIds.forEach(newCloudId -> { PACloud paCloud = em.find(PACloud.class, newCloudId); + List blacklistedRegions = Arrays.asList(paCloud.getBlacklist().split(",")); JSONArray images = connectorIaasGateway.getImages(paCloud.getDummyInfrastructureName()); + List consolidatedImages = images.toList().parallelStream().map(o -> (JSONObject) o).filter(record -> !blacklistedRegions.contains(record.get("location"))).collect(Collectors.toList()); //TODO: (Optimization) An images per region map structure could be the best here. // It can reduce the getNodeCandidates calls to PA. - images.forEach(img -> { - JSONObject image = (JSONObject) img; + consolidatedImages.forEach(image -> { String region = image.optString("location"); String imageReq; switch (paCloud.getCloudProviderName()) { -- GitLab From 42019709fd5424559748d115c095e0e3d5faa924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20COMPASTI=C3=89?= Date: Tue, 22 Dec 2020 20:13:06 +0100 Subject: [PATCH 7/7] minor fix --- src/main/java/org/activeeon/morphemic/model/PACloud.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/activeeon/morphemic/model/PACloud.java b/src/main/java/org/activeeon/morphemic/model/PACloud.java index 8de7793f..93e86f76 100644 --- a/src/main/java/org/activeeon/morphemic/model/PACloud.java +++ b/src/main/java/org/activeeon/morphemic/model/PACloud.java @@ -51,7 +51,7 @@ public class PACloud implements Serializable { @Column(name = "DUMMY_INFRASTRUCTURE_NAME") private String dummyInfrastructureName; - @Column(name = "DEFAULTNETWORK") + @Column(name = "DEFAULT_NETWORK") private String defaultNetwork; @Column(name = "BLACKLIST") -- GitLab