From e2f391e875dd6bea608b30fab85afa7565f480c3 Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Sat, 26 Jun 2021 05:20:57 +0200 Subject: [PATCH 01/12] Adding the BYON NS xml file --- .../target/classes/Define_NS_BYON.xml | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml diff --git a/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml b/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml new file mode 100644 index 00000000..cd1a94e6 --- /dev/null +++ b/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + 508 + + + 510.5 + + + + + + + + + + + + + + + + ]]> + + + \ No newline at end of file -- GitLab From 714e33bf39a6b946f8b4e26c75970271f034379e Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Sat, 26 Jun 2021 05:44:19 +0200 Subject: [PATCH 02/12] adding registerNewByonNodes and defineByonNodeSource functions --- .../org/activeeon/morphemic/PAGateway.java | 96 +++++++++++++++---- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index 35567aa2..78683e14 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -456,6 +456,62 @@ public class PAGateway { TemporaryFilesHelper.delete(fXmlFile); } + /** + * Define a BYON node source + * @param byonNodeList a list of BYON nodes to be connected to the server. + * @param nodeSourceName The name of the node source + */ + public void defineByonNodeSource(List byonNodeList, String nodeSourceName) { + Map variables = new HashMap<>(); + String filename; + String byonIPs = ""; + // Prepare the ip addresses for all the nodes to be added + for(ByonNode byonNode: byonNodeList){ + List tempListIP=byonNode.getIpAddresses(); + assert !tempListIP.isEmpty(); + byonIPs= byonIPs + tempListIP.get(0).getValue() + ","; + } + // Collect the pamr router address and port number + try { + URL endpointPa = (new URL(this.paURL)); + variables.put("rm_host_name", endpointPa.getHost()); + variables.put("pa_port", "" + endpointPa.getPort()); + } catch (MalformedURLException e) { + LOGGER.error(String.valueOf(e.getStackTrace())); + } + /* IMPORTANT: we consider that all of the byon nodes that belong to the same nodesource are going to have the + same info and as a result we will get all the nodesource info from one node, later this should be improved. + TODO */ + assert !byonNodeList.isEmpty(); + ByonNode byonNode = byonNodeList.get(0); + filename = File.separator + "Define_NS_BYON.xml"; + variables.put("NS_name", nodeSourceName); + variables.put("pa_protocol", "http"); + variables.put("tokens", "ALL"); + variables.put("ssh_username", byonNode.getLoginCredential().getUsername()); + variables.put("ssh_password", byonNode.getLoginCredential().getPassword()); + /* IMPORTANT: Later we should relay only on the ssh_key. For now all the nodes must have the same login + credentials. later by automating the node deployment process we can add the server key automatically. + TODO */ + variables.put("ssh_key", ""); + variables.put("ssh_port", "22"); + variables.put("list_of_ips", byonIPs); + // Create the xml file + File fXmlFile = null; + LOGGER.info("NodeSource deployment workflow filename: " + filename); + try { + fXmlFile = TemporaryFilesHelper.createTempFileFromResource(filename); + } catch (IOException ioe) { + LOGGER.error("Opening the NS deployment workflow file failed due to : " + Arrays.toString(ioe.getStackTrace())); + } + assert fXmlFile != null; + LOGGER.info("Submitting the file: " + fXmlFile.toString()); + LOGGER.info("Trying to deploy the NS: " + nodeSourceName); + JobId jobId = schedulerGateway.submit(fXmlFile, variables); + LOGGER.info("Job submitted with ID: " + jobId); + TemporaryFilesHelper.delete(fXmlFile); + } + /** * Add an EMS deployment to a defined job * @param nodeNames Names of the nodes to which to add EMS deployment @@ -562,25 +618,33 @@ public class PAGateway { /** * Register new BYON nodes passed as ByonDefinition object - * @param byonNodeDefinition an object of class ByonDefinition that contains the detials of the nodes to be registered. + * @param byonNodeDefinitionList objects of class ByonDefinition that contains the detials of the nodes to be registered. * @param jobId A constructed job identifier - * @return ByonNode an object that contains information about the registered Node + * @return ByonNodeList objects of class ByonNode that contains information about the registered Node */ - public ByonNode registerNewByonNode(ByonDefinition byonNodeDefinition, String jobId) { - Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); + public List registerNewByonNodes(List byonNodeDefinitionList, String jobId) { + assert !byonNodeDefinitionList.isEmpty(); + String nodeSourceName="BYON_TEST"; Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered."); - - EntityManagerHelper.begin(); - - ByonNode newByonNode = new ByonNode(); - newByonNode.setName(byonNodeDefinition.getName()); - newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); - newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); - newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); - - EntityManagerHelper.persist(newByonNode); - EntityManagerHelper.commit(); - return newByonNode; + List ByonNodeList = new LinkedList<>(); + for(ByonDefinition byonNodeDefinition: byonNodeDefinitionList) { + Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); + + EntityManagerHelper.begin(); + // Convert the Definition to Node + ByonNode newByonNode = new ByonNode(); + newByonNode.setName(byonNodeDefinition.getName()); + newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); + newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); + newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); + // Register the Node in the database + EntityManagerHelper.persist(newByonNode); + EntityManagerHelper.commit(); + // Add the node to the list to be returned + ByonNodeList.add(newByonNode); + } + defineByonNodeSource(ByonNodeList,nodeSourceName); + return ByonNodeList; //TODO } -- GitLab From 083261dc992b19adb15209f67186cd6e3848d14c Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Wed, 7 Jul 2021 16:12:51 +0200 Subject: [PATCH 03/12] implementing addByonNodes --- .../org/activeeon/morphemic/PAGateway.java | 76 ++++++++++++------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index 78683e14..c6188a9c 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -8,6 +8,7 @@ import org.activeeon.morphemic.infrastructure.deployment.PAConnectorIaasGateway; import org.activeeon.morphemic.infrastructure.deployment.PAResourceManagerGateway; import org.activeeon.morphemic.model.*; import org.activeeon.morphemic.service.*; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.Validate; import org.json.JSONArray; import org.json.JSONObject; @@ -618,34 +619,29 @@ public class PAGateway { /** * Register new BYON nodes passed as ByonDefinition object - * @param byonNodeDefinitionList objects of class ByonDefinition that contains the detials of the nodes to be registered. + * @param byonNodeDefinition objects of class ByonDefinition that contains the detials of the nodes to be registered. * @param jobId A constructed job identifier * @return ByonNodeList objects of class ByonNode that contains information about the registered Node */ - public List registerNewByonNodes(List byonNodeDefinitionList, String jobId) { - assert !byonNodeDefinitionList.isEmpty(); - String nodeSourceName="BYON_TEST"; + public ByonNode registerNewByonNode(ByonDefinition byonNodeDefinition, String jobId) { + Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered."); - List ByonNodeList = new LinkedList<>(); - for(ByonDefinition byonNodeDefinition: byonNodeDefinitionList) { - Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); - - EntityManagerHelper.begin(); - // Convert the Definition to Node - ByonNode newByonNode = new ByonNode(); - newByonNode.setName(byonNodeDefinition.getName()); - newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); - newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); - newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); - // Register the Node in the database - EntityManagerHelper.persist(newByonNode); - EntityManagerHelper.commit(); - // Add the node to the list to be returned - ByonNodeList.add(newByonNode); - } - defineByonNodeSource(ByonNodeList,nodeSourceName); - return ByonNodeList; - //TODO + + EntityManagerHelper.begin(); + + ByonNode newByonNode = new ByonNode(); + newByonNode.setName(byonNodeDefinition.getName()); + newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); + newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); + newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); + EntityManagerHelper.persist(newByonNode); + EntityManagerHelper.commit(); + + return newByonNode; + /*TODO: + * Avoid duplicate nodes in the database + */ + } /** @@ -654,8 +650,8 @@ public class PAGateway { * @return List of ByonNode objects that contains information about the registered Nodes */ public List getByonNodeList(String jobId) { - return null; - //TODO + List listByonNodes = EntityManagerHelper.createQuery("SELECT byon FROM ByonNode byon", ByonNode.class).getResultList(); + return listByonNodes; } /** @@ -665,8 +661,18 @@ public class PAGateway { * @return 0 if nodes has been added properly. A greater than 0 value otherwise. */ public int addByonNodes(Map byonIdPerComponent, String jobId) { + for (Map.Entry entry : byonIdPerComponent.entrySet()) { + ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, entry.getKey()); + List byonNodeList = new LinkedList<>(); + byonNodeList.add(byonNode); + defineByonNodeSource(byonNodeList, byonNode.getName()); + } return 0; - //TODO + /*TODO: + * Create node sources per JobID + * Set NodeSource name should be BYON_ + * Create mapping in the DB between the nodes and the components + * */ } /** @@ -764,8 +770,20 @@ public class PAGateway { * @return true if the deletion was done with no errors, false otherwise */ public Boolean deleteByonNode(String byonId) { - return null; - //TODO + EntityManagerHelper.begin(); + ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, byonId); + if (byonNode == null) { + return false; + } + else { + System.out.println("Deleting the node: " + byonNode.getId()); + EntityManagerHelper.remove(byonNode); + EntityManagerHelper.commit(); + return true; + } + /*TODO: + * add logging info when deleting + * */ } /** -- GitLab From 8fac111091b6b4d15e1e8f9b53d65587f41374ec Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Thu, 8 Jul 2021 06:51:01 +0200 Subject: [PATCH 04/12] Adding the getByonNodeList method --- .../org/activeeon/morphemic/PAGateway.java | 18 +++++++++++++++--- .../activeeon/morphemic/model/ByonNode.java | 7 +++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index c6188a9c..b78b7876 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -626,7 +626,7 @@ public class PAGateway { public ByonNode registerNewByonNode(ByonDefinition byonNodeDefinition, String jobId) { Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered."); - + Job job = EntityManagerHelper.find(Job.class, jobId); EntityManagerHelper.begin(); ByonNode newByonNode = new ByonNode(); @@ -634,6 +634,7 @@ public class PAGateway { newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); + newByonNode.setJob(job); EntityManagerHelper.persist(newByonNode); EntityManagerHelper.commit(); @@ -646,12 +647,23 @@ public class PAGateway { /** * Return the List of registered BYON nodes - * @param jobId A constructed job identifier + * @param jobId A constructed job identifier, If "0" is passed as the JobId all the Byon Nodes will be returned * @return List of ByonNode objects that contains information about the registered Nodes */ public List getByonNodeList(String jobId) { + List filteredByonNodes = new LinkedList<>(); List listByonNodes = EntityManagerHelper.createQuery("SELECT byon FROM ByonNode byon", ByonNode.class).getResultList(); - return listByonNodes; + if(jobId.equals("0")) {return listByonNodes;} + else{ + for (ByonNode byonNode : listByonNodes){ + if (byonNode.getJob().getJobId().equals(jobId)){ + filteredByonNodes.add(byonNode);} + } + return filteredByonNodes; + } + /*TODO: + * Add Logging info + */ } /** diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java index 17cb40ac..847e249f 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java @@ -2,7 +2,9 @@ package org.activeeon.morphemic.model; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; import org.hibernate.annotations.GenericGenerator; import javax.persistence.*; @@ -61,6 +63,11 @@ public class ByonNode implements Serializable { @JsonProperty("allocated") private Boolean allocated = null; + @Setter + @Getter + @ManyToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH) + private Job job; + public ByonNode name(String name) { this.name = name; return this; -- GitLab From 3c27d182b6c21271e9b316f4802318bfd702a907 Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Mon, 19 Jul 2021 10:36:53 +0200 Subject: [PATCH 05/12] Add BYON Node Candidates and Cloud --- .../org/activeeon/morphemic/PAGateway.java | 79 ++++++++++++++++++- .../activeeon/morphemic/model/ByonNode.java | 75 +----------------- .../activeeon/morphemic/model/CloudType.java | 2 + 3 files changed, 82 insertions(+), 74 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index b78b7876..8f2cfc50 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -627,24 +627,97 @@ public class PAGateway { Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered."); Job job = EntityManagerHelper.find(Job.class, jobId); + // assert job is not null EntityManagerHelper.begin(); - + NodeCandidate byonNC = createByonNodeCandidate(byonNodeDefinition, jobId); ByonNode newByonNode = new ByonNode(); newByonNode.setName(byonNodeDefinition.getName()); newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); newByonNode.setIpAddresses(byonNodeDefinition.getIpAddresses()); newByonNode.setNodeProperties(byonNodeDefinition.getNodeProperties()); newByonNode.setJob(job); + newByonNode.setNodeCandidate(byonNC); + EntityManagerHelper.persist(newByonNode); EntityManagerHelper.commit(); return newByonNode; - /*TODO: - * Avoid duplicate nodes in the database + /* TODO: + * Avoid duplicate nodes in the database + * Create the NC here + */ + + } + + /** + * Create a dummy object of class Cloud to be used for the node candidates + * @return the created byonCloud object + */ + private Cloud createByonCloud() { + EntityManagerHelper.begin(); + //Check if the Byon cloud already exists + List clouds =EntityManagerHelper.createQuery("SELECT cloud FROM Cloud cloud", Cloud.class).getResultList();; + for(Cloud cloud : clouds ) { + if(cloud.getCloudType()==CloudType.BYON && cloud.getOwner().equals("BYON") && cloud.getId().equals("byon")) { + // Halt, the Byon Cloud already exist + return cloud;} + } + + //else, Byon cloud will be created + Cloud byonCloud = new Cloud(); + byonCloud.setCloudType(CloudType.BYON); + byonCloud.setOwner("BYON"); + byonCloud.setId("byon"); + + //Add the Byon cloud to the database + EntityManagerHelper.persist(byonCloud); + EntityManagerHelper.commit(); + return byonCloud; + + /* TODO : + * Check if we have to add other variables to the new cloud */ + } + /** + * @param byonDef an Object of class ByonDefinition that contains all the nodes Definition + * @return an object of class NodeCandidate + */ + public NodeCandidate createByonNodeCandidate( ByonDefinition byonDef, String jobId) { + //create a dummy cloud + Cloud byoncloud = createByonCloud(); + //set the image + Image image = new Image(); + image.setOperatingSystem(byonDef.getNodeProperties().getOperatingSystem()); + //image.setId("Byon_Image_"+jobId); + //set the hardware + Hardware hardware = new Hardware(); + hardware.setCores(byonDef.getNodeProperties().getNumberOfCores()); + hardware.setDisk((double) byonDef.getNodeProperties().getDisk()); + hardware.setRam(byonDef.getNodeProperties().getMemory()); + //hardware.setId("ByonHardware_"+jobId); + //set the location + Location location = new Location(); + location.setGeoLocation(byonDef.getNodeProperties().getGeoLocation()); + //location.setId("Byon_Location_"+jobId); + + //define a dummy NC + NodeCandidate byonNC = new NodeCandidate(); + //byonNC.setId("Byon_NC_"+jobId); + byonNC.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.BYON); + byonNC.setJobIdForBYON(jobId); + byonNC.setPrice(0.0); + byonNC.setMemoryPrice(0.0); + byonNC.setPricePerInvocation(0.0); + byonNC.setCloud(byoncloud); + byonNC.setImage(image); + byonNC.setHardware(hardware); + byonNC.setLocation(location); + return byonNC; } + + /** * Return the List of registered BYON nodes * @param jobId A constructed job identifier, If "0" is passed as the JobId all the Byon Nodes will be returned diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java index 847e249f..89fce2ca 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/ByonNode.java @@ -19,6 +19,8 @@ import java.util.Objects; @AllArgsConstructor @NoArgsConstructor @Entity +@Getter +@Setter @Table(name = "BYON_NODE") public class ByonNode implements Serializable { @Id @@ -51,9 +53,8 @@ public class ByonNode implements Serializable { @JsonProperty("diagnostic") private String diagnostic = null; - @Column(name = "NODE_CANDIDATE") - @JsonProperty("nodeCandidate") - private String nodeCandidate = null; + @OneToOne(fetch = FetchType.EAGER, orphanRemoval = true, cascade = CascadeType.REFRESH) + private NodeCandidate nodeCandidate = null; @Column(name = "USER_ID") @JsonProperty("userId") @@ -63,8 +64,6 @@ public class ByonNode implements Serializable { @JsonProperty("allocated") private Boolean allocated = null; - @Setter - @Getter @ManyToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH) private Job job; @@ -149,85 +148,24 @@ public class ByonNode implements Serializable { return this; } - /** - * Reason this node was created - * @return reason - **/ - public String getReason() { - return reason; - } - public void setReason(String reason) { - this.reason = reason; - } public ByonNode diagnostic(String diagnostic) { this.diagnostic = diagnostic; return this; } - /** - * Diagnostic information about the node state - * @return diagnostic - **/ - public String getDiagnostic() { - return diagnostic; - } - - public void setDiagnostic(String diagnostic) { - this.diagnostic = diagnostic; - } - - public ByonNode nodeCandidate(String nodeCandidate) { - this.nodeCandidate = nodeCandidate; - return this; - } - - /** - * The node candidate this node was created from if applicable. - * @return nodeCandidate - **/ - public String getNodeCandidate() { - return nodeCandidate; - } - - public void setNodeCandidate(String nodeCandidate) { - this.nodeCandidate = nodeCandidate; - } public ByonNode id(String id) { this.id = id; return this; } - /** - * Unique identifier of this BYON. - * @return id - **/ - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - public ByonNode userId(String userId) { this.userId = userId; return this; } - /** - * User id of the owner of this node. - * @return userId - **/ - public String getUserId() { - return userId; - } - - public void setUserId(String userId) { - this.userId = userId; - } public ByonNode allocated(Boolean allocated) { this.allocated = allocated; @@ -242,11 +180,6 @@ public class ByonNode implements Serializable { return allocated; } - public void setAllocated(Boolean allocated) { - this.allocated = allocated; - } - - @Override public boolean equals(java.lang.Object o) { if (this == o) { diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/CloudType.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/CloudType.java index 731e19bd..f9fafdeb 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/CloudType.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/CloudType.java @@ -14,6 +14,8 @@ public enum CloudType { PUBLIC("PUBLIC"), + BYON("BYON"), + SIMULATION("SIMULATION"); private String value; -- GitLab From 15225d02415a3491459c65c51bba48743ae5bc2c Mon Sep 17 00:00:00 2001 From: mklkun Date: Mon, 19 Jul 2021 13:00:34 +0200 Subject: [PATCH 06/12] Fix BYON node candidate and cloud creation --- .../org/activeeon/morphemic/PAGateway.java | 80 ++--------------- .../morphemic/model/NodeCandidate.java | 7 ++ .../morphemic/service/ByonUtils.java | 89 +++++++++++++++++++ 3 files changed, 104 insertions(+), 72 deletions(-) create mode 100644 scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ByonUtils.java diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index 8f2cfc50..8f956f39 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -8,7 +8,7 @@ import org.activeeon.morphemic.infrastructure.deployment.PAConnectorIaasGateway; import org.activeeon.morphemic.infrastructure.deployment.PAResourceManagerGateway; import org.activeeon.morphemic.model.*; import org.activeeon.morphemic.service.*; -import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.Validate; import org.json.JSONArray; import org.json.JSONObject; @@ -344,7 +344,8 @@ public class PAGateway { List allNodeCandidates = EntityManagerHelper.createQuery("SELECT nc FROM NodeCandidate nc", NodeCandidate.class).getResultList(); allNodeCandidates.forEach(nodeCandidate -> { - if (JCloudsInstancesUtils.isHandledHardwareInstanceType(nodeCandidate.getCloud().getApi().getProviderName(), + if (nodeCandidate.isByonNodeCandidate() || + JCloudsInstancesUtils.isHandledHardwareInstanceType(nodeCandidate.getCloud().getApi().getProviderName(), nodeCandidate.getHardware().getName())) { if (NodeCandidateUtils.verifyAllFilters(requirements, nodeCandidate)) { filteredNodeCandidates.add(nodeCandidate); @@ -627,9 +628,11 @@ public class PAGateway { Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered."); Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered."); Job job = EntityManagerHelper.find(Job.class, jobId); - // assert job is not null + assert job != null; + + LOGGER.info("Registering a new BYON definition related to job " + jobId + " ..."); + NodeCandidate byonNC = ByonUtils.createByonNodeCandidate(byonNodeDefinition, jobId); EntityManagerHelper.begin(); - NodeCandidate byonNC = createByonNodeCandidate(byonNodeDefinition, jobId); ByonNode newByonNode = new ByonNode(); newByonNode.setName(byonNodeDefinition.getName()); newByonNode.setLoginCredential(byonNodeDefinition.getLoginCredential()); @@ -640,6 +643,7 @@ public class PAGateway { EntityManagerHelper.persist(newByonNode); EntityManagerHelper.commit(); + LOGGER.info("BYON node registered."); return newByonNode; /* TODO: @@ -649,74 +653,6 @@ public class PAGateway { } - /** - * Create a dummy object of class Cloud to be used for the node candidates - * @return the created byonCloud object - */ - private Cloud createByonCloud() { - EntityManagerHelper.begin(); - //Check if the Byon cloud already exists - List clouds =EntityManagerHelper.createQuery("SELECT cloud FROM Cloud cloud", Cloud.class).getResultList();; - for(Cloud cloud : clouds ) { - if(cloud.getCloudType()==CloudType.BYON && cloud.getOwner().equals("BYON") && cloud.getId().equals("byon")) { - // Halt, the Byon Cloud already exist - return cloud;} - } - - //else, Byon cloud will be created - Cloud byonCloud = new Cloud(); - byonCloud.setCloudType(CloudType.BYON); - byonCloud.setOwner("BYON"); - byonCloud.setId("byon"); - - //Add the Byon cloud to the database - EntityManagerHelper.persist(byonCloud); - EntityManagerHelper.commit(); - return byonCloud; - - /* TODO : - * Check if we have to add other variables to the new cloud - */ - } - - /** - * @param byonDef an Object of class ByonDefinition that contains all the nodes Definition - * @return an object of class NodeCandidate - */ - public NodeCandidate createByonNodeCandidate( ByonDefinition byonDef, String jobId) { - //create a dummy cloud - Cloud byoncloud = createByonCloud(); - //set the image - Image image = new Image(); - image.setOperatingSystem(byonDef.getNodeProperties().getOperatingSystem()); - //image.setId("Byon_Image_"+jobId); - //set the hardware - Hardware hardware = new Hardware(); - hardware.setCores(byonDef.getNodeProperties().getNumberOfCores()); - hardware.setDisk((double) byonDef.getNodeProperties().getDisk()); - hardware.setRam(byonDef.getNodeProperties().getMemory()); - //hardware.setId("ByonHardware_"+jobId); - //set the location - Location location = new Location(); - location.setGeoLocation(byonDef.getNodeProperties().getGeoLocation()); - //location.setId("Byon_Location_"+jobId); - - //define a dummy NC - NodeCandidate byonNC = new NodeCandidate(); - //byonNC.setId("Byon_NC_"+jobId); - byonNC.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.BYON); - byonNC.setJobIdForBYON(jobId); - byonNC.setPrice(0.0); - byonNC.setMemoryPrice(0.0); - byonNC.setPricePerInvocation(0.0); - byonNC.setCloud(byoncloud); - byonNC.setImage(image); - byonNC.setHardware(hardware); - byonNC.setLocation(location); - return byonNC; - } - - /** * Return the List of registered BYON nodes diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java index fd461b17..5211ef69 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java @@ -288,6 +288,13 @@ public class NodeCandidate implements Serializable { this.environment = environment; } + /** + * Check if a node candidate is of BYON type + * @return true if yes, false if not + */ + public boolean isByonNodeCandidate() { + return nodeCandidateType.value.equals("BYON"); + } @Override public boolean equals(java.lang.Object o) { diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ByonUtils.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ByonUtils.java new file mode 100644 index 00000000..a9f774fc --- /dev/null +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ByonUtils.java @@ -0,0 +1,89 @@ +package org.activeeon.morphemic.service; + +import lombok.extern.slf4j.Slf4j; +import org.activeeon.morphemic.model.*; +import org.apache.commons.lang3.RandomStringUtils; + +import java.util.Optional; + +@Slf4j +public class ByonUtils { + + private ByonUtils() {} + + /** + * @param byonDef an Object of class ByonDefinition that contains all the nodes Definition + * @return an object of class NodeCandidate + */ + public static NodeCandidate createByonNodeCandidate(ByonDefinition byonDef, String jobId) { + LOGGER.debug("Creating the BYON node candidate ..."); + //create a dummy cloud + Cloud byonCloud = ByonUtils.getOrCreateDummyByonCloud(); + EntityManagerHelper.begin(); + //set the image + Image image = new Image(); + image.setId("byon-image-" + RandomStringUtils.randomAlphanumeric(16)); + image.setOperatingSystem(byonDef.getNodeProperties().getOperatingSystem()); + + //set the hardware + Hardware hardware = new Hardware(); + hardware.setId("byon-hardware-" + RandomStringUtils.randomAlphanumeric(16)); + hardware.setCores(byonDef.getNodeProperties().getNumberOfCores()); + hardware.setDisk((double) byonDef.getNodeProperties().getDisk()); + hardware.setRam(byonDef.getNodeProperties().getMemory()); + + //set the location + Location location = new Location(); + location.setId("byon-location-" + RandomStringUtils.randomAlphanumeric(16)); + location.setGeoLocation(byonDef.getNodeProperties().getGeoLocation()); + + //define a dummy NC + NodeCandidate byonNC = new NodeCandidate(); + byonNC.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.BYON); + byonNC.setJobIdForBYON(jobId); + byonNC.setPrice(0.0); + byonNC.setMemoryPrice(0.0); + byonNC.setPricePerInvocation(0.0); + byonNC.setCloud(byonCloud); + byonNC.setImage(image); + byonNC.setHardware(hardware); + byonNC.setLocation(location); + + EntityManagerHelper.persist(byonNC); + EntityManagerHelper.commit(); + LOGGER.info("BYON node candidate created."); + return byonNC; + } + + /** + * Create a dummy object of class Cloud to be used for the node candidates + * @return the created byonCloud object + */ + public static Cloud getOrCreateDummyByonCloud() { + LOGGER.debug("Searching for the dummy BYON cloud ..."); + EntityManagerHelper.begin(); + //Check if the Byon cloud already exists + Optional optCloud = Optional.ofNullable(EntityManagerHelper.find(Cloud.class, "byon")); + if (optCloud.isPresent()) { + LOGGER.info("Dummy BYON cloud found!"); + return optCloud.get(); + } + + LOGGER.debug("Creating the dummy BYON cloud ..."); + //else, Byon cloud will be created + Cloud byonCloud = new Cloud(); + byonCloud.setCloudType(CloudType.BYON); + byonCloud.setOwner("BYON"); + byonCloud.setId("byon"); + + //Add the Byon cloud to the database + EntityManagerHelper.persist(byonCloud); + EntityManagerHelper.commit(); + LOGGER.info("Dummy BYON cloud created."); + return byonCloud; + + /* TODO : + * Check if we have to add other variables to the new cloud + */ + } +} -- GitLab From 7e984be6891e570b8d6a86cd2a48d8091b6d9f96 Mon Sep 17 00:00:00 2001 From: mklkun Date: Mon, 26 Jul 2021 11:56:10 +0200 Subject: [PATCH 07/12] Add BYON deployment, persistence and scaling support --- .../org/activeeon/morphemic/PAGateway.java | 141 ++++++++++++++---- .../activeeon/morphemic/model/Deployment.java | 8 + .../morphemic/model/NodeCandidate.java | 2 +- .../morphemic/model/NodeTypeRequirement.java | 2 +- .../src/main/resources/Define_NS_BYON.xml | 129 ++++++++++++++++ .../resources/acquire_node_BYON_script.groovy | 34 +++++ 6 files changed, 286 insertions(+), 30 deletions(-) create mode 100644 scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml create mode 100644 scheduling-abstraction-layer/src/main/resources/acquire_node_BYON_script.groovy diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index 8f956f39..9990ce33 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -8,7 +8,6 @@ import org.activeeon.morphemic.infrastructure.deployment.PAConnectorIaasGateway; import org.activeeon.morphemic.infrastructure.deployment.PAResourceManagerGateway; import org.activeeon.morphemic.model.*; import org.activeeon.morphemic.service.*; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.Validate; import org.json.JSONArray; import org.json.JSONObject; @@ -581,6 +580,7 @@ public class PAGateway { Deployment newDeployment = new Deployment(); JSONObject nodeCandidateInfo = node.optJSONObject("nodeCandidateInformation"); newDeployment.setNodeName(node.optString("nodeName")); + newDeployment.setDeploymentType(NodeType.IAAS); newDeployment.setLocationName(nodeCandidateInfo.optString("locationName")); newDeployment.setImageProviderId(nodeCandidateInfo.optString("imageProviderId")); newDeployment.setHardwareProviderId(nodeCandidateInfo.optString("hardwareProviderId")); @@ -662,11 +662,13 @@ public class PAGateway { public List getByonNodeList(String jobId) { List filteredByonNodes = new LinkedList<>(); List listByonNodes = EntityManagerHelper.createQuery("SELECT byon FROM ByonNode byon", ByonNode.class).getResultList(); - if(jobId.equals("0")) {return listByonNodes;} - else{ - for (ByonNode byonNode : listByonNodes){ - if (byonNode.getJob().getJobId().equals(jobId)){ - filteredByonNodes.add(byonNode);} + if (jobId.equals("0")) { + return listByonNodes; + } else { + for (ByonNode byonNode : listByonNodes) { + if (byonNode.getJob().getJobId().equals(jobId)) { + filteredByonNodes.add(byonNode); + } } return filteredByonNodes; } @@ -682,12 +684,37 @@ public class PAGateway { * @return 0 if nodes has been added properly. A greater than 0 value otherwise. */ public int addByonNodes(Map byonIdPerComponent, String jobId) { - for (Map.Entry entry : byonIdPerComponent.entrySet()) { - ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, entry.getKey()); + Validate.notNull(byonIdPerComponent, "The received byonIdPerComponent structure is empty. Nothing to be added."); + + EntityManagerHelper.begin(); + + byonIdPerComponent.forEach((byonNodeId, componentId) -> { + ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, byonNodeId); + Task task = EntityManagerHelper.find(Task.class, componentId); + + Deployment newDeployment = new Deployment(); + newDeployment.setNodeName(byonNode.getName()); + newDeployment.setDeploymentType(NodeType.BYON); + newDeployment.setByonNode(byonNode); + List byonNodeList = new LinkedList<>(); byonNodeList.add(byonNode); defineByonNodeSource(byonNodeList, byonNode.getName()); - } + LOGGER.info("BYON node source defined."); + + newDeployment.setTask(task); + newDeployment.setNumber(task.getNextDeploymentID()); + EntityManagerHelper.persist(newDeployment); + LOGGER.debug("Deployment created: " + newDeployment.toString()); + + task.addDeployment(newDeployment); + EntityManagerHelper.persist(task); + }); + + EntityManagerHelper.commit(); + + LOGGER.info("BYON nodes added properly."); + return 0; /*TODO: * Create node sources per JobID @@ -981,15 +1008,26 @@ public class PAGateway { } private ScriptTask createInfraTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { + switch (deployment.getDeploymentType()) { + case IAAS: + return createInfraIAASTask(task, deployment, taskNameSuffix, nodeToken); + case BYON: + return createInfraBYONTask(task, deployment, taskNameSuffix, nodeToken); + } + + return new ScriptTask(); + } + + private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { LOGGER.debug("Acquiring node AWS script file: " + getClass().getResource(File.separator + "acquire_node_aws_script.groovy").toString()); ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() + taskNameSuffix, - "acquire_node_aws_script.groovy"); + "acquire_node_aws_script.groovy"); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle("pre_acquire_node_script.groovy", "groovy")); Map variablesMap = new HashMap<>(); variablesMap.put("NS_name", new TaskVariable("NS_name", - deployment.getPaCloud().getNodeSourceNamePrefix() + deployment.getLocationName())); + deployment.getPaCloud().getNodeSourceNamePrefix() + deployment.getLocationName())); variablesMap.put("nVMs", new TaskVariable("nVMs", "1", "PA:Integer", false)); variablesMap.put("synchronous", new TaskVariable("synchronous", "true", "PA:Boolean", false)); variablesMap.put("timeout", new TaskVariable("timeout", "700", "PA:Long", false)); @@ -1011,24 +1049,57 @@ public class PAGateway { LOGGER.debug("Variables to be added to the task: " + variablesMap.toString()); deployNodeTask.setVariables(variablesMap); + addLocalDefaultNSRegexSelectionScript(deployNodeTask); + + return deployNodeTask; + } + + private ScriptTask createInfraBYONTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { + LOGGER.debug("Acquiring node BYON script file: " + getClass().getResource(File.separator + "acquire_node_BYON_script.groovy").toString()); + ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireBYONNode_" + task.getName() + taskNameSuffix, + "acquire_node_BYON_script.groovy"); + + deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle("pre_acquire_node_script.groovy", "groovy")); + + Map variablesMap = new HashMap<>(); + variablesMap.put("NS_name", new TaskVariable("NS_name", + deployment.getPaCloud().getNodeSourceNamePrefix() + deployment.getLocationName())); + //TODO: To check this. Which ip address is used as host in RM? + variablesMap.put("host_name", new TaskVariable("host_name", + deployment.getByonNode().getIpAddresses().get(0).getValue())); + variablesMap.put("token", new TaskVariable("token", nodeToken)); + + LOGGER.debug("Variables to be added to the task: " + variablesMap.toString()); + deployNodeTask.setVariables(variablesMap); + + addLocalDefaultNSRegexSelectionScript(deployNodeTask); + + return deployNodeTask; + } + + private void addLocalDefaultNSRegexSelectionScript(ScriptTask scriptTask) { try { String selectionScriptFileName = "check_node_source_regexp.groovy"; String[] nodeSourceNameRegex = {"^local$|^Default$"}; SelectionScript selectionScript = new SelectionScript(Utils.getContentWithFileName(selectionScriptFileName), - "groovy", - nodeSourceNameRegex, - true); - deployNodeTask.setSelectionScript(selectionScript); + "groovy", + nodeSourceNameRegex, + true); + scriptTask.setSelectionScript(selectionScript); } catch (InvalidScriptException e) { LOGGER.warn("Selection script could not have been added."); } - - return deployNodeTask; } private ScriptTask createEmsDeploymentTask(EmsDeploymentRequest emsDeploymentRequest, String taskNameSuffix, String nodeToken) { LOGGER.debug("Preparing EMS deployment task"); - ScriptTask emsDeploymentTask = PAFactory.createComplexScriptTaskFromFiles("emsDeployment" + taskNameSuffix,"emsdeploy_mainscript.groovy","groovy","emsdeploy_prescript.sh","bash","emsdeploy_postscript.sh","bash"); + ScriptTask emsDeploymentTask = PAFactory.createComplexScriptTaskFromFiles("emsDeployment" + taskNameSuffix, + "emsdeploy_mainscript.groovy", + "groovy", + "emsdeploy_prescript.sh", + "bash", + "emsdeploy_postscript.sh", + "bash"); Map variablesMap = emsDeploymentRequest.getWorkflowMap(); emsDeploymentTask.addGenericInformation("NODE_ACCESS_TOKEN", nodeToken); emsDeploymentTask.setVariables(variablesMap); @@ -1064,7 +1135,7 @@ public class PAGateway { // Let's retrieve the deployment to clone if (optTask.get().getDeployments() == null || optTask.get().getDeployments().isEmpty()) { - LOGGER.error(String.format("No previous deployment found in task [%s] ",taskName)); + LOGGER.error(String.format("No previous deployment found in task [%s] ", taskName)); return 2; } @@ -1073,6 +1144,10 @@ public class PAGateway { // Let's clone the deployment/node as needed Deployment oldDeployment = optTask.get().getDeployments().get(0); + if (NodeType.BYON.equals(oldDeployment.getDeploymentType())) { + LOGGER.error(String.format("The previous deployment is a BYON node [%s] ", oldDeployment)); + return 3; + } nodeNames.stream().map(nodeName -> { EmsDeploymentRequest newEmsDeploymentReq = oldDeployment.getEmsDeployment() == null ? null : oldDeployment.getEmsDeployment().clone(nodeName); @@ -1085,6 +1160,8 @@ public class PAGateway { oldDeployment.getTask(), false, null, + null, + NodeType.IAAS, null ); }) @@ -1374,10 +1451,24 @@ public class PAGateway { } // For supplied node, I retrieve their deployment - List deployments = nodeNames.stream().map(node -> EntityManagerHelper.find(Deployment.class,node)).filter(Objects::nonNull).collect(Collectors.toList()); + List deployments = nodeNames.stream().map(node -> EntityManagerHelper.find(Deployment.class,node)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + deployments = deployments.stream().filter(deployment -> { + if (NodeType.BYON.equals(deployment.getDeploymentType())) { + LOGGER.warn("Deployment " + deployment.getNodeName() + " is a BYON node and can't be removed in scaling in."); + return false; + } + return true; + }).collect(Collectors.toList()); // For deployed node, I flag their removal - List nodesToBeRemoved = deployments.stream().filter(Deployment::getIsDeployed).map(Deployment::getNodeName).collect(Collectors.toList()); + List nodesToBeRemoved = deployments.stream() + .filter(Deployment::getIsDeployed) + .map(Deployment::getNodeName) + .collect(Collectors.toList()); + LOGGER.info("Nodes to be removed are : " + nodesToBeRemoved); // For every node, I remove the deployment entree deployments.forEach( @@ -1584,13 +1675,7 @@ public class PAGateway { jobToSubmit.getTasks().forEach(task -> { List scriptTasks = buildPATask(task, jobToSubmit); - scriptTasks.forEach(scriptTask -> { - try { - paJob.addTask(scriptTask); - } catch (UserException e) { - LOGGER.error("Task " + task.getName() + " could not be added due to: " + e.toString()); - } - }); + addAllScriptTasksToPAJob(paJob, task, scriptTasks); EntityManagerHelper.persist(task); }); diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java index d19b9f9b..1584338f 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java @@ -47,6 +47,13 @@ public class Deployment implements Serializable { @Column(name = "NUMBER") private Long number; + @Column(name = "NODE_TYPE") + @Enumerated(EnumType.STRING) + private NodeType deploymentType; + + @OneToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH) + private ByonNode byonNode; + @Override public String toString() { return "Deployment{" + @@ -59,6 +66,7 @@ public class Deployment implements Serializable { ", number='" + number + '\'' + ", paCloud='" + paCloud.getNodeSourceNamePrefix() + '\'' + ", task='" + task.getName() + '\'' + + ", byonNode='" + byonNode.toString() + '\'' + '}'; } } diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java index 5211ef69..f0dcfd2a 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeCandidate.java @@ -71,7 +71,7 @@ public class NodeCandidate implements Serializable { private NodeCandidateTypeEnum nodeCandidateType = null; @Column(name = "JOB_ID_FOR_BYON") - @JsonProperty("nodeType") + @JsonProperty("jobIdForByon") private String jobIdForBYON; @Column(name = "PRICE") diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeTypeRequirement.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeTypeRequirement.java index 4b74245f..a9355cb1 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeTypeRequirement.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/NodeTypeRequirement.java @@ -16,7 +16,7 @@ public class NodeTypeRequirement extends Requirement { @JsonProperty("nodeType") private List nodeTypes; - @JsonProperty("nodeType") + @JsonProperty("jobIdForByon") private String jobIdForBYON; /** diff --git a/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml b/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml new file mode 100644 index 00000000..cd1a94e6 --- /dev/null +++ b/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + 508 + + + 510.5 + + + + + + + + + + + + + + + + ]]> + + + \ No newline at end of file diff --git a/scheduling-abstraction-layer/src/main/resources/acquire_node_BYON_script.groovy b/scheduling-abstraction-layer/src/main/resources/acquire_node_BYON_script.groovy new file mode 100644 index 00000000..6591d203 --- /dev/null +++ b/scheduling-abstraction-layer/src/main/resources/acquire_node_BYON_script.groovy @@ -0,0 +1,34 @@ +import org.ow2.proactive.resourcemanager.common.event.RMNodeEvent + +// Conncting to the Scheduler +println "[+] Preparation of Nodes ... " +print "(1/4) Connecting to the RM ..." +rmapi.connect() +println " ... OK !" + +// Getting NS configuration settings +def retCode = 0; +def nodeSourceName = variables.get("NS_name") +def nodeToken = variables.get("token") +def hostName = variables.get("host_name") + +// Enforcing .... +print "(2/4) Searching for nodes ..." +def nodeURLs = rmapi.getRMStateFull().getNodesEvents().findAll { nodeEvent -> + return (nodeSourceName == nodeEvent.getNodeSource() && hostName == nodeEvent.getHostName()); +}.collect { nodeEvent -> + return (nodeEvent.getNodeUrl()); +} +println " ... OK !" +println "Found node URLs : " + nodeURLs.toString() + +print "(3/4) Acquiring nodes ..." +nodeURLs.each { nodeURL -> + rmapi.addNodeToken(nodeURL, nodeToken) +} +println " ... OK !" + +print "(4/4) Logging out ..." +rmapi.disconnect(); +println " ... OK !" +return retCode; \ No newline at end of file -- GitLab From cea2c2034ffa3f20a91cceadcb35be05e5e13732 Mon Sep 17 00:00:00 2001 From: mklkun Date: Mon, 26 Jul 2021 12:00:51 +0200 Subject: [PATCH 08/12] Delete file staged in build folder --- .../target/classes/Define_NS_BYON.xml | 129 ------------------ 1 file changed, 129 deletions(-) delete mode 100644 scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml diff --git a/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml b/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml deleted file mode 100644 index cd1a94e6..00000000 --- a/scheduling-abstraction-layer/target/classes/Define_NS_BYON.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - 508 - - - 510.5 - - - - - - - - - - - - - - - - ]]> - - - \ No newline at end of file -- GitLab From b7cf01227ef321d8d412ab4a19e2a776d5fae700 Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Mon, 26 Jul 2021 17:09:18 +0200 Subject: [PATCH 09/12] create a token for BYON nodes when NS is defined --- .../org/activeeon/morphemic/PAGateway.java | 8 ++--- .../src/main/resources/Define_NS_BYON.xml | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index 9990ce33..d4b9f750 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -488,7 +488,7 @@ public class PAGateway { filename = File.separator + "Define_NS_BYON.xml"; variables.put("NS_name", nodeSourceName); variables.put("pa_protocol", "http"); - variables.put("tokens", "ALL"); + variables.put("tokens", "BYON_"+byonNode.getJob().getJobId()); variables.put("ssh_username", byonNode.getLoginCredential().getUsername()); variables.put("ssh_password", byonNode.getLoginCredential().getPassword()); /* IMPORTANT: Later we should relay only on the ssh_key. For now all the nodes must have the same login @@ -648,7 +648,6 @@ public class PAGateway { return newByonNode; /* TODO: * Avoid duplicate nodes in the database - * Create the NC here */ } @@ -687,7 +686,6 @@ public class PAGateway { Validate.notNull(byonIdPerComponent, "The received byonIdPerComponent structure is empty. Nothing to be added."); EntityManagerHelper.begin(); - byonIdPerComponent.forEach((byonNodeId, componentId) -> { ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, byonNodeId); Task task = EntityManagerHelper.find(Task.class, componentId); @@ -699,7 +697,7 @@ public class PAGateway { List byonNodeList = new LinkedList<>(); byonNodeList.add(byonNode); - defineByonNodeSource(byonNodeList, byonNode.getName()); + defineByonNodeSource(byonNodeList, "BYON_NS_"+jobId); LOGGER.info("BYON node source defined."); newDeployment.setTask(task); @@ -718,8 +716,6 @@ public class PAGateway { return 0; /*TODO: * Create node sources per JobID - * Set NodeSource name should be BYON_ - * Create mapping in the DB between the nodes and the components * */ } diff --git a/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml b/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml index cd1a94e6..5e471570 100644 --- a/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml +++ b/scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml @@ -3,15 +3,15 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:proactive:jobdescriptor:3.12" xsi:schemaLocation="urn:proactive:jobdescriptor:3.12 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.12/schedulerjob.xsd" name="Define_NS_BYON.xml" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" > - - - - - - - - - + + + + + + + + + @@ -51,9 +51,13 @@ def hosts = "" for (ip in ips) { hosts = hosts + ip + " 1\n" } + if (tokens=="") { tokens="ALL" } +else { + tokens="tokens="+tokens +} println "[+] Preparation of NodeSoure " + nodeSourceName print "(1/4) Connecting to the RM at "+protocol+"://"+host+":"+port+"/ ..." @@ -97,10 +101,10 @@ return retCode; - 508 + 511 - 510.5 + 514.5 @@ -115,12 +119,12 @@ return retCode; left:0 !important; top:0 !important; width:2726px; - height:3116px; + height:3360px; } - + ]]> -- GitLab From 0a865d6c4518afa4d1f95daace06a325e9e2192a Mon Sep 17 00:00:00 2001 From: mklkun Date: Tue, 27 Jul 2021 18:45:44 +0200 Subject: [PATCH 10/12] Fix Deployment toString() according to node type --- .../activeeon/morphemic/model/Deployment.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java index 1584338f..dccfd404 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java @@ -56,17 +56,35 @@ public class Deployment implements Serializable { @Override public String toString() { - return "Deployment{" + - "nodeName='" + nodeName + '\'' + - ", locationName='" + locationName + '\'' + - ", imageProviderId='" + imageProviderId + '\'' + - ", hardwareProviderId='" + hardwareProviderId + '\'' + - ", isDeployed='" + isDeployed.toString() + '\'' + - ", nodeAccessToken='" + nodeAccessToken + '\'' + - ", number='" + number + '\'' + - ", paCloud='" + paCloud.getNodeSourceNamePrefix() + '\'' + - ", task='" + task.getName() + '\'' + - ", byonNode='" + byonNode.toString() + '\'' + - '}'; + switch (deploymentType) { + case IAAS: + return "Deployment{" + + "nodeName='" + nodeName + '\'' + + ", locationName='" + locationName + '\'' + + ", imageProviderId='" + imageProviderId + '\'' + + ", hardwareProviderId='" + hardwareProviderId + '\'' + + ", isDeployed='" + isDeployed.toString() + '\'' + + ", nodeAccessToken='" + nodeAccessToken + '\'' + + ", number='" + number + '\'' + + ", paCloud='" + paCloud.getNodeSourceNamePrefix() + '\'' + + ", task='" + task.getName() + '\'' + + ", byonNode='" + byonNode + '\'' + + '}'; + case BYON: + return "Deployment{" + + "nodeName='" + nodeName + '\'' + + ", locationName='" + locationName + '\'' + + ", imageProviderId='" + imageProviderId + '\'' + + ", hardwareProviderId='" + hardwareProviderId + '\'' + + ", isDeployed='" + isDeployed.toString() + '\'' + + ", nodeAccessToken='" + nodeAccessToken + '\'' + + ", number='" + number + '\'' + + ", paCloud='" + paCloud + '\'' + + ", task='" + task.getName() + '\'' + + ", byonNode='" + byonNode.toString() + '\'' + + '}'; + default: + return "Deployment{nodeName='" + nodeName + '}'; + } } } -- GitLab From cd014c46acf92d96a900e375b255035cdbfb23c8 Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Thu, 29 Jul 2021 02:37:40 +0200 Subject: [PATCH 11/12] limit the log for BYON deployment --- .../src/main/java/org/activeeon/morphemic/model/Deployment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java index dccfd404..d2fe50eb 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java @@ -81,7 +81,7 @@ public class Deployment implements Serializable { ", number='" + number + '\'' + ", paCloud='" + paCloud + '\'' + ", task='" + task.getName() + '\'' + - ", byonNode='" + byonNode.toString() + '\'' + + ", byonNode='" + byonNode.getName() + '\'' + '}'; default: return "Deployment{nodeName='" + nodeName + '}'; -- GitLab From 9eda6b26f9d54adbf07de84d1ae694ab88c3d9ff Mon Sep 17 00:00:00 2001 From: Ali_fahs Date: Thu, 29 Jul 2021 03:06:45 +0200 Subject: [PATCH 12/12] adding the undeploy and remove NS function --- .../org/activeeon/morphemic/PAGateway.java | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java index d4b9f750..729ab1c2 100644 --- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java +++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -690,6 +690,9 @@ public class PAGateway { ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, byonNodeId); Task task = EntityManagerHelper.find(Task.class, componentId); + assert byonNode!=null : "The BYON ID passed in the mapping does not exist in the database"; + assert task!=null : "The componentId passed in the mapping does not exist in the database"; + Deployment newDeployment = new Deployment(); newDeployment.setNodeName(byonNode.getName()); newDeployment.setDeploymentType(NodeType.BYON); @@ -697,8 +700,9 @@ public class PAGateway { List byonNodeList = new LinkedList<>(); byonNodeList.add(byonNode); - defineByonNodeSource(byonNodeList, "BYON_NS_"+jobId); - LOGGER.info("BYON node source defined."); + LOGGER.info("BYON node Added: " + byonNode.getName() + " Ip: " + byonNode.getIpAddresses().get(0).getValue() ); + defineByonNodeSource(byonNodeList, "BYON_NS_"+byonNode.getId()); + LOGGER.info("BYON node source BYON_NS_"+ byonNode.getId() + " is defined"); newDeployment.setTask(task); newDeployment.setNumber(task.getNextDeploymentID()); @@ -712,13 +716,10 @@ public class PAGateway { EntityManagerHelper.commit(); LOGGER.info("BYON nodes added properly."); - return 0; - /*TODO: - * Create node sources per JobID - * */ } + /** * Undeploy clouds * @param cloudIDs List of cloud IDs to remove @@ -817,19 +818,59 @@ public class PAGateway { EntityManagerHelper.begin(); ByonNode byonNode = EntityManagerHelper.find(ByonNode.class, byonId); if (byonNode == null) { + LOGGER.error("The passed BYON ID is not Found in the database"); return false; } else { - System.out.println("Deleting the node: " + byonNode.getId()); + if(!undeployByonNs(byonNode, false, true)) { + LOGGER.warn("The BYON node source undeploy finished with errors"); + } + LOGGER.info("Deleting the BYON node " + byonNode.getId() +" from the database" ); EntityManagerHelper.remove(byonNode); EntityManagerHelper.commit(); - return true; } + return true; /*TODO: - * add logging info when deleting + * change the hardcoding for preempt and remove variables * */ } + /** + * Undeploy or remove the node source of a BYON node + * @param byonNode an object of class ByonNode to be undeployed or removed. + * @param preempt If true undeploy or remove node source immediately without waiting for nodes to be freed + * @param remove If true completely remove the node source, if false only undeply the node source + * @return true if the resourceManagerGateway return no errors, false otherwise + */ + private Boolean undeployByonNs(ByonNode byonNode, Boolean preempt, Boolean remove) { + assert byonNode!= null : "A null value was passed for byonNode, A node source must have a BYON ID"; + String nodeSourceName = "BYON_NS_" + byonNode.getId(); + if (remove) { + try { + LOGGER.info("Removing BYON node source " + nodeSourceName + " from the ProActive server"); + resourceManagerGateway.removeNodeSource(nodeSourceName, preempt); + } catch (NotConnectedException | PermissionRestException e) { + LOGGER.error(Arrays.toString(e.getStackTrace())); + return false; + } + } + else { + try { + LOGGER.info("Undeploying BYON node source " + nodeSourceName + " from the ProActive server"); + resourceManagerGateway.undeployNodeSource(nodeSourceName, preempt); + } catch (NotConnectedException | PermissionRestException e) { + LOGGER.error(Arrays.toString(e.getStackTrace())); + return false; + } + } + LOGGER.info("BYON node source was removed with no errors"); + return true; + + /*TODO: + * Check if the nodes source is present before calling the resourceManagerGateway + */ + } + /** * Stop jobs * @param jobIDs List of job IDs to stop -- GitLab