Skip to content
GitLab
Commit a7ee733d authored by maciek riedl's avatar maciek riedl
Browse files

Merge branch 'add-edge-support' into 'morphemic-rc2.5' 

Add edge support

See merge request !285
parents 51bd6600 20da4ad1
Pipeline #20714 passed with stages
in 34 minutes and 37 seconds
Hide whitespace changes
Inline Side-by-side
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java
View file @ a7ee733d
......@@ -536,6 +536,74 @@ public class PAGateway {
TemporaryFilesHelper.delete(fXmlFile);
}
/**
* Define a EDGE node source
* @param edgeNodeList a list of EDGE nodes to be connected to the server.
* @param nodeSourceName The name of the node source
*/
public void defineEdgeNodeSource(List<EdgeNode> edgeNodeList, String nodeSourceName) {
Map<String, String> variables = new HashMap<>();
String filename;
String EdgeIPs = "";
// Prepare the ip addresses for all the nodes to be added
for(EdgeNode edgeNode: edgeNodeList){
List<IpAddress> tempListIP=edgeNode.getIpAddresses();
assert !tempListIP.isEmpty();
EdgeIPs= EdgeIPs + tempListIP.get(0).getValue().replace(" ", "") + ",";
}
// Collect the pamr router address and port number
try {
URL endpointPa = (new URL(this.paURL));
variables.put("rm_host_name", endpointPa.getHost());
variables.put("pa_port", "" + endpointPa.getPort());
} catch (MalformedURLException e) {
LOGGER.error(String.valueOf(e.getStackTrace()));
}
assert !edgeNodeList.isEmpty();
EdgeNode edgeNode = edgeNodeList.get(0);
filename = File.separator + "Define_NS_EDGE.xml";
variables.put("NS_name", nodeSourceName);
variables.put("pa_protocol", "http");
variables.put("tokens", "EDGE_" + edgeNode.getJobId());
variables.put("ssh_username", edgeNode.getLoginCredential().getUsername());
variables.put("ssh_password", edgeNode.getLoginCredential().getPassword());
/* IMPORTANT: Later we should relay only on the ssh_key. For now all the nodes must have the same login
credentials. later by automating the node deployment process we can add the server key automatically.
TODO */
variables.put("ssh_key", "");
variables.put("ssh_port", "22");
variables.put("list_of_ips", EdgeIPs);
switch (edgeNode.getSystemArch()) {
case "AMD":
variables.put("deployment_mode", "useStartupScript");
variables.put("script_url", edgeNode.getScriptURL());
variables.put("script_path", "/tmp/proactive-agent.sh");
break;
case "ARM":
variables.put("deployment_mode", "useNodeJarStartupScript");
variables.put("jar_url", edgeNode.getJarURL());
break;
default:
LOGGER.error("The Edge node system architecture {} is not supported!", edgeNode.getSystemArch());
}
// Create the xml file
File fXmlFile = null;
LOGGER.info("NodeSource deployment workflow filename: " + filename);
try {
fXmlFile = TemporaryFilesHelper.createTempFileFromResource(filename);
} catch (IOException ioe) {
LOGGER.error("Opening the NS deployment workflow file failed due to : " + Arrays.toString(ioe.getStackTrace()));
}
assert fXmlFile != null;
LOGGER.info("Submitting the file: " + fXmlFile.toString());
LOGGER.info("Trying to deploy the NS: " + nodeSourceName);
JobId jobId = schedulerGateway.submit(fXmlFile, variables);
LOGGER.info("Job submitted with ID: " + jobId);
TemporaryFilesHelper.delete(fXmlFile);
}
/**
* Add an EMS deployment to a defined job
* @param nodeNames Names of the nodes to which to add EMS deployment
......@@ -656,14 +724,14 @@ public class PAGateway {
* @param byonNodeDefinition objects of class ByonDefinition that contains the detials of the nodes to be registered.
* @param jobId A constructed job identifier
* @param Automate the Byon agent will be deployed automatically if the value is set to True
* @return ByonNodeList objects of class ByonNode that contains information about the registered Node
* @return newByonNode object of class ByonNode that contains information about the registered Node
*/
public ByonNode registerNewByonNode(ByonDefinition byonNodeDefinition, String jobId, boolean Automate) {
Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered.");
Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered.");
LOGGER.info("registerNewByonNode endpoint is called with Automate set to " + Automate
+ ", Registering a new BYON definition related to job " + jobId + " ...");
NodeCandidate byonNC = ByonUtils.createByonNodeCandidate(byonNodeDefinition, jobId);
NodeCandidate byonNC = ByonUtils.createNodeCandidate(byonNodeDefinition.getNodeProperties(), jobId, "byon");
EntityManagerHelper.begin();
ByonNode newByonNode = new ByonNode();
newByonNode.setName(byonNodeDefinition.getName());
......@@ -687,6 +755,42 @@ public class PAGateway {
}
/**
* Register new Edge nodes passed as EdgeDefinition object
*
* @param edgeNodeDefinition objects of class ByonDefinition that contains the detials of the nodes to be registered.
* @param jobId A constructed job identifier
* @return newEdgeNode object of class EdgeNode that contains information about the registered Node
*/
public EdgeNode registerNewEdgeNode(EdgeDefinition edgeNodeDefinition, String jobId) {
Validate.notNull(edgeNodeDefinition, "The received EDGE node definition is empty. Nothing to be registered.");
Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered.");
LOGGER.info("registerNewEdgeNode endpoint is called, Registering a new EDGE definition related to job " + jobId + " ...");
EntityManagerHelper.begin();
EdgeNode newEdgeNode = new EdgeNode();
newEdgeNode.setName(edgeNodeDefinition.getName());
newEdgeNode.setLoginCredential(edgeNodeDefinition.getLoginCredential());
newEdgeNode.setIpAddresses(edgeNodeDefinition.getIpAddresses());
newEdgeNode.setNodeProperties(edgeNodeDefinition.getNodeProperties());
newEdgeNode.setJobId(jobId);
newEdgeNode.setSystemArch(edgeNodeDefinition.getSystemArch());
newEdgeNode.setScriptURL(edgeNodeDefinition.getScriptURL());
newEdgeNode.setJarURL(edgeNodeDefinition.getJarURL());
NodeCandidate edgeNC = ByonUtils.createNodeCandidate(edgeNodeDefinition.getNodeProperties(), jobId, "edge");
newEdgeNode.setNodeCandidate(edgeNC);
EntityManagerHelper.persist(newEdgeNode);
EntityManagerHelper.commit();
LOGGER.info("EDGE node registered.");
return newEdgeNode;
/* TODO:
* Avoid duplicate nodes in the database
*/
}
/**
* Return the List of registered BYON nodes
......@@ -711,6 +815,31 @@ public class PAGateway {
*/
}
/**
* Return the List of registered EDGE nodes
* @param jobId A constructed job identifier, If "0" is passed as the JobId all the Edge Nodes will be returned
* @return List of EdgeNode objects that contains information about the registered Nodes
*/
public List<EdgeNode> getEdgeNodeList(String jobId) {
List<EdgeNode> filteredEdgeNodes = new LinkedList<>();
List<EdgeNode> listEdgeNodes = EntityManagerHelper.createQuery("SELECT edge FROM EdgeNode edge", EdgeNode.class).getResultList();
if (jobId.equals("0")) {
return listEdgeNodes;
} else {
for (EdgeNode edgeNode : listEdgeNodes) {
if (jobId.equals(edgeNode.getJobId())) {
filteredEdgeNodes.add(edgeNode);
}
}
return filteredEdgeNodes;
}
/*TODO:
* Add Logging info
*/
}
/**
* Adding BYON nodes to a job component
* @param byonIdPerComponent a mapping between byon nodes and job components
......@@ -763,6 +892,59 @@ public class PAGateway {
return 0;
}
/**
* Adding EDGE nodes to a job component
* @param edgeIdPerComponent a mapping between byon nodes and job components
* @param jobId A constructed job identifier
* @return 0 if nodes has been added properly. A greater than 0 value otherwise.
*/
public int addEdgeNodes(Map<String, String> edgeIdPerComponent, String jobId) {
Validate.notNull(edgeIdPerComponent, "The received byonIdPerComponent structure is empty. Nothing to be added.");
EntityManagerHelper.begin();
edgeIdPerComponent.forEach((edgeNodeId, componentName) -> {
EdgeNode edgeNode = EntityManagerHelper.find(EdgeNode.class, edgeNodeId);
Task task = EntityManagerHelper.find(Task.class, jobId + componentName);
assert edgeNode!=null : "The EDGE ID passed in the mapping does not exist in the database";
assert task!=null : "The componentId passed in the mapping does not exist in the database";
Deployment newDeployment = new Deployment();
newDeployment.setNodeName(edgeNode.getName());
newDeployment.setDeploymentType(NodeType.EDGE);
newDeployment.setEdgeNode(edgeNode);
PACloud cloud = new PACloud();
String nodeSourceName = "EDGE_NS_" + edgeNode.getId();
cloud.setCloudID(nodeSourceName);
cloud.setNodeSourceNamePrefix(nodeSourceName);
cloud.setCloudType(CloudType.EDGE);
cloud.addDeployment(newDeployment);
newDeployment.setPaCloud(cloud);
EntityManagerHelper.persist(cloud);
List<EdgeNode> edgeNodeList = new LinkedList<>();
edgeNodeList.add(edgeNode);
LOGGER.info("EDGE node Added: " + edgeNode.getName() + " Ip: " + edgeNode.getIpAddresses().get(0).getValue());
defineEdgeNodeSource(edgeNodeList, nodeSourceName);
LOGGER.info("EDGE node source EDGE_NS_" + edgeNode.getId() + " is defined");
newDeployment.setTask(task);
newDeployment.setNumber(task.getNextDeploymentID());
EntityManagerHelper.persist(newDeployment);
LOGGER.debug("Deployment created: " + newDeployment.toString());
task.addDeployment(newDeployment);
EntityManagerHelper.persist(task);
});
EntityManagerHelper.commit();
LOGGER.info("EDGE nodes added properly.");
return 0;
}
/**
* Undeploy clouds
* @param cloudIDs List of cloud IDs to remove
......@@ -1127,6 +1309,7 @@ public class PAGateway {
null,
null,
NodeType.IAAS,
null,
null
);
})
......
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/CloudType.java
View file @ a7ee733d
......@@ -16,6 +16,8 @@ public enum CloudType {
BYON("BYON"),
EDGE("EDGE"),
SIMULATION("SIMULATION");
private String value;
......
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/Deployment.java
View file @ a7ee733d
......@@ -62,6 +62,10 @@ public class Deployment implements Serializable {
@OneToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH)
private ByonNode byonNode;
@OneToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH)
private EdgeNode edgeNode;
public static void clean() {
List<Deployment> allDeployments = EntityManagerHelper.createQuery("SELECT d FROM Deployment d", Deployment.class).getResultList();
allDeployments.forEach(EntityManagerHelper::remove);
......@@ -101,6 +105,22 @@ public class Deployment implements Serializable {
", task='" + task.getName() + '\'' +
", byonNode='" + byonNode.getName() + '\'' +
'}';
case EDGE:
return "Deployment{" +
"nodeName='" + nodeName + '\'' +
", locationName='" + locationName + '\'' +
", imageProviderId='" + imageProviderId + '\'' +
", hardwareProviderId='" + hardwareProviderId + '\'' +
", isDeployed='" + isDeployed.toString() + '\'' +
", instanceId='" + instanceId + '\'' +
", ipAddress='" + ipAddress + '\'' +
", nodeAccessToken='" + nodeAccessToken + '\'' +
", number='" + number + '\'' +
", paCloud='" + paCloud + '\'' +
", task='" + task.getName() + '\'' +
", edgeNode='" + edgeNode.getName() + '\'' +
'}';
default:
return "Deployment{nodeName='" + nodeName + '}';
}
......
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/EdgeDefinition.java 0 → 100644
View file @ a7ee733d
package org.activeeon.morphemic.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.*;
import java.util.List;
/**
* Attributes defining a BYON node
*/
@AllArgsConstructor
@NoArgsConstructor
@Getter
@Setter
@ToString(callSuper = true)
public class EdgeDefinition {
@JsonProperty("name")
private String name = null;
@JsonProperty("systemArch")
private String systemArch = null;
@JsonProperty("scriptURL")
private String scriptURL = null;
@JsonProperty("jarURL")
private String jarURL = null;
@JsonProperty("loginCredential")
private LoginCredential loginCredential = null;
@JsonProperty("ipAddresses")
private List<IpAddress> ipAddresses = null;
@JsonProperty("nodeProperties")
private NodeProperties nodeProperties = null;
}
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/model/EdgeNode.java 0 → 100644
View file @ a7ee733d
package org.activeeon.morphemic.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.activeeon.morphemic.service.EntityManagerHelper;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* Representation of an Edge used by ProActive
* This class is a clone of the Byon Node Class with some modifications
*/
@AllArgsConstructor
@NoArgsConstructor
@Entity
@Getter
@Setter
@Table(name = "EDGE_NODE")
public class EdgeNode implements Serializable {
@Id
@GeneratedValue(generator = "system-uuid")
@GenericGenerator(name = "system-uuid", strategy = "uuid")
@Column(name = "ID")
@JsonProperty("id")
private String id = null;
@Column(name = "NAME")
@JsonProperty("name")
private String name = null;
@Column(name = "SYSTEM_ARCH")
@JsonProperty("systemArch")
private String systemArch = null;
@JsonProperty("scriptURL")
private String scriptURL = null;
@JsonProperty("scriptURL")
private String jarURL = null;
@Embedded
@JsonProperty("loginCredential")
private LoginCredential loginCredential = null;
@ElementCollection(targetClass=IpAddress.class)
private List<IpAddress> ipAddresses = null;
@Embedded
@JsonProperty("nodeProperties")
private NodeProperties nodeProperties = null;
@Column(name = "REASON")
@JsonProperty("reason")
private String reason = null;
@Column(name = "DIAGNOSTIC")
@JsonProperty("diagnostic")
private String diagnostic = null;
@OneToOne(fetch = FetchType.EAGER, orphanRemoval = true, cascade = CascadeType.REFRESH)
private NodeCandidate nodeCandidate = null;
@Column(name = "USER_ID")
@JsonProperty("userId")
private String userId = null;
@Column(name = "ALLOCATED")
@JsonProperty("allocated")
private Boolean allocated = null;
@Column(name = "JOB_ID")
@JsonProperty("jobId")
private String jobId;
public static void clean() {
List<EdgeNode> allEdgeNodes = EntityManagerHelper.createQuery("SELECT en FROM EdgeNode en", EdgeNode.class).getResultList();
allEdgeNodes.forEach(EntityManagerHelper::remove);
}
public EdgeNode name(String name) {
this.name = name;
return this;
}
/**
* Human-readable name for the node.
* @return name
**/
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public EdgeNode loginCredential(LoginCredential loginCredential) {
this.loginCredential = loginCredential;
return this;
}
/**
* Get loginCredential
* @return loginCredential
**/
public LoginCredential getLoginCredential() {
return loginCredential;
}
public void setLoginCredential(LoginCredential loginCredential) {
this.loginCredential = loginCredential;
}
public EdgeNode ipAddresses(List<IpAddress> ipAddresses) {
this.ipAddresses = ipAddresses;
return this;
}
public EdgeNode addIpAddressesItem(IpAddress ipAddressesItem) {
if (this.ipAddresses == null) {
this.ipAddresses = new ArrayList<IpAddress>();
}
this.ipAddresses.add(ipAddressesItem);
return this;
}
/**
* The public/private ip addresses under which this node is reachable.
* @return ipAddresses
**/
public List<IpAddress> getIpAddresses() {
return ipAddresses;
}
public void setIpAddresses(List<IpAddress> ipAddresses) {
this.ipAddresses = ipAddresses;
}
public EdgeNode nodeProperties(NodeProperties nodeProperties) {
this.nodeProperties = nodeProperties;
return this;
}
/**
* Further properties of this node.
* @return nodeProperties
**/
public NodeProperties getNodeProperties() {
return nodeProperties;
}
public void setNodeProperties(NodeProperties nodeProperties) {
this.nodeProperties = nodeProperties;
}
public EdgeNode reason(String reason) {
this.reason = reason;
return this;
}
public EdgeNode diagnostic(String diagnostic) {
this.diagnostic = diagnostic;
return this;
}
public EdgeNode id(String id) {
this.id = id;
return this;
}
public EdgeNode userId(String userId) {
this.userId = userId;
return this;
}
public EdgeNode allocated(Boolean allocated) {
this.allocated = allocated;
return this;
}
/**
* Signals if the node was allocated by cloudiator
* @return allocated
**/
public Boolean isAllocated() {
return allocated;
}
@Override
public boolean equals(java.lang.Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
EdgeNode edgeNode = (EdgeNode) o;
return Objects.equals(this.name, edgeNode.name) &&
Objects.equals(this.loginCredential, edgeNode.loginCredential) &&
Objects.equals(this.ipAddresses, edgeNode.ipAddresses) &&
Objects.equals(this.nodeProperties, edgeNode.nodeProperties) &&
Objects.equals(this.reason, edgeNode.reason) &&
Objects.equals(this.diagnostic, edgeNode.diagnostic) &&
Objects.equals(this.nodeCandidate, edgeNode.nodeCandidate) &&
Objects.equals(this.id, edgeNode.id) &&
Objects.equals(this.userId, edgeNode.userId) &&
Objects.equals(this.allocated, edgeNode.allocated) &&
Objects.equals(this.jobId, edgeNode.jobId);
}
@Override
public int hashCode() {
return Objects.hash(name, loginCredential, ipAddresses, nodeProperties, reason, diagnostic, nodeCandidate, id, userId, allocated);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("class EdgeNode {\n");
sb.append(" name: ").append(toIndentedString(name)).append("\n");
sb.append(" loginCredential: ").append(toIndentedString(loginCredential)).append("\n");
sb.append(" ipAddresses: ").append(toIndentedString(ipAddresses)).append("\n");
sb.append(" nodeProperties: ").append(toIndentedString(nodeProperties)).append("\n");
sb.append(" reason: ").append(toIndentedString(reason)).append("\n");
sb.append(" diagnostic: ").append(toIndentedString(diagnostic)).append("\n");
sb.append(" nodeCandidate: ").append(toIndentedString(nodeCandidate)).append("\n");
sb.append(" id: ").append(toIndentedString(id)).append("\n");
sb.append(" userId: ").append(toIndentedString(userId)).append("\n");
sb.append(" allocated: ").append(toIndentedString(allocated)).append("\n");
sb.append(" jobId: ").append(toIndentedString(jobId)).append("\n");
sb.append("}");
return sb.toString();
}