Skip to content
GitLab
Commit 20da4ad1 authored by alijawadfahs's avatar alijawadfahs
Browse files

adding the edge endpoints

parent 750c18b8
Hide whitespace changes
Inline Side-by-side
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/PAGateway.java
View file @ 20da4ad1
......@@ -536,6 +536,74 @@ public class PAGateway {
TemporaryFilesHelper.delete(fXmlFile);
}
/**
* Define a EDGE node source
* @param edgeNodeList a list of EDGE nodes to be connected to the server.
* @param nodeSourceName The name of the node source
*/
public void defineEdgeNodeSource(List<EdgeNode> edgeNodeList, String nodeSourceName) {
Map<String, String> variables = new HashMap<>();
String filename;
String EdgeIPs = "";
// Prepare the ip addresses for all the nodes to be added
for(EdgeNode edgeNode: edgeNodeList){
List<IpAddress> tempListIP=edgeNode.getIpAddresses();
assert !tempListIP.isEmpty();
EdgeIPs= EdgeIPs + tempListIP.get(0).getValue().replace(" ", "") + ",";
}
// Collect the pamr router address and port number
try {
URL endpointPa = (new URL(this.paURL));
variables.put("rm_host_name", endpointPa.getHost());
variables.put("pa_port", "" + endpointPa.getPort());
} catch (MalformedURLException e) {
LOGGER.error(String.valueOf(e.getStackTrace()));
}
assert !edgeNodeList.isEmpty();
EdgeNode edgeNode = edgeNodeList.get(0);
filename = File.separator + "Define_NS_EDGE.xml";
variables.put("NS_name", nodeSourceName);
variables.put("pa_protocol", "http");
variables.put("tokens", "EDGE_" + edgeNode.getJobId());
variables.put("ssh_username", edgeNode.getLoginCredential().getUsername());
variables.put("ssh_password", edgeNode.getLoginCredential().getPassword());
/* IMPORTANT: Later we should relay only on the ssh_key. For now all the nodes must have the same login
credentials. later by automating the node deployment process we can add the server key automatically.
TODO */
variables.put("ssh_key", "");
variables.put("ssh_port", "22");
variables.put("list_of_ips", EdgeIPs);
switch (edgeNode.getSystemArch()) {
case "AMD":
variables.put("deployment_mode", "useStartupScript");
variables.put("script_url", edgeNode.getScriptURL());
variables.put("script_path", "/tmp/proactive-agent.sh");
break;
case "ARM":
variables.put("deployment_mode", "useNodeJarStartupScript");
variables.put("jar_url", edgeNode.getJarURL());
break;
default:
LOGGER.error("The Edge node system architecture {} is not supported!", edgeNode.getSystemArch());
}
// Create the xml file
File fXmlFile = null;
LOGGER.info("NodeSource deployment workflow filename: " + filename);
try {
fXmlFile = TemporaryFilesHelper.createTempFileFromResource(filename);
} catch (IOException ioe) {
LOGGER.error("Opening the NS deployment workflow file failed due to : " + Arrays.toString(ioe.getStackTrace()));
}
assert fXmlFile != null;
LOGGER.info("Submitting the file: " + fXmlFile.toString());
LOGGER.info("Trying to deploy the NS: " + nodeSourceName);
JobId jobId = schedulerGateway.submit(fXmlFile, variables);
LOGGER.info("Job submitted with ID: " + jobId);
TemporaryFilesHelper.delete(fXmlFile);
}
/**
* Add an EMS deployment to a defined job
* @param nodeNames Names of the nodes to which to add EMS deployment
......@@ -656,14 +724,14 @@ public class PAGateway {
* @param byonNodeDefinition objects of class ByonDefinition that contains the detials of the nodes to be registered.
* @param jobId A constructed job identifier
* @param Automate the Byon agent will be deployed automatically if the value is set to True
* @return ByonNodeList objects of class ByonNode that contains information about the registered Node
* @return newByonNode object of class ByonNode that contains information about the registered Node
*/
public ByonNode registerNewByonNode(ByonDefinition byonNodeDefinition, String jobId, boolean Automate) {
Validate.notNull(byonNodeDefinition, "The received Byon node definition is empty. Nothing to be registered.");
Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered.");
LOGGER.info("registerNewByonNode endpoint is called with Automate set to " + Automate
+ ", Registering a new BYON definition related to job " + jobId + " ...");
NodeCandidate byonNC = ByonUtils.createByonNodeCandidate(byonNodeDefinition, jobId);
NodeCandidate byonNC = ByonUtils.createNodeCandidate(byonNodeDefinition.getNodeProperties(), jobId, "byon");
EntityManagerHelper.begin();
ByonNode newByonNode = new ByonNode();
newByonNode.setName(byonNodeDefinition.getName());
......@@ -687,6 +755,42 @@ public class PAGateway {
}
/**
* Register new Edge nodes passed as EdgeDefinition object
*
* @param edgeNodeDefinition objects of class ByonDefinition that contains the detials of the nodes to be registered.
* @param jobId A constructed job identifier
* @return newEdgeNode object of class EdgeNode that contains information about the registered Node
*/
public EdgeNode registerNewEdgeNode(EdgeDefinition edgeNodeDefinition, String jobId) {
Validate.notNull(edgeNodeDefinition, "The received EDGE node definition is empty. Nothing to be registered.");
Validate.notNull(jobId, "The received jobId is empty. Nothing to be registered.");
LOGGER.info("registerNewEdgeNode endpoint is called, Registering a new EDGE definition related to job " + jobId + " ...");
EntityManagerHelper.begin();
EdgeNode newEdgeNode = new EdgeNode();
newEdgeNode.setName(edgeNodeDefinition.getName());
newEdgeNode.setLoginCredential(edgeNodeDefinition.getLoginCredential());
newEdgeNode.setIpAddresses(edgeNodeDefinition.getIpAddresses());
newEdgeNode.setNodeProperties(edgeNodeDefinition.getNodeProperties());
newEdgeNode.setJobId(jobId);
newEdgeNode.setSystemArch(edgeNodeDefinition.getSystemArch());
newEdgeNode.setScriptURL(edgeNodeDefinition.getScriptURL());
newEdgeNode.setJarURL(edgeNodeDefinition.getJarURL());
NodeCandidate edgeNC = ByonUtils.createNodeCandidate(edgeNodeDefinition.getNodeProperties(), jobId, "edge");
newEdgeNode.setNodeCandidate(edgeNC);
EntityManagerHelper.persist(newEdgeNode);
EntityManagerHelper.commit();
LOGGER.info("EDGE node registered.");
return newEdgeNode;
/* TODO:
* Avoid duplicate nodes in the database
*/
}
/**
* Return the List of registered BYON nodes
......@@ -711,6 +815,31 @@ public class PAGateway {
*/
}
/**
* Return the List of registered EDGE nodes
* @param jobId A constructed job identifier, If "0" is passed as the JobId all the Edge Nodes will be returned
* @return List of EdgeNode objects that contains information about the registered Nodes
*/
public List<EdgeNode> getEdgeNodeList(String jobId) {
List<EdgeNode> filteredEdgeNodes = new LinkedList<>();
List<EdgeNode> listEdgeNodes = EntityManagerHelper.createQuery("SELECT edge FROM EdgeNode edge", EdgeNode.class).getResultList();
if (jobId.equals("0")) {
return listEdgeNodes;
} else {
for (EdgeNode edgeNode : listEdgeNodes) {
if (jobId.equals(edgeNode.getJobId())) {
filteredEdgeNodes.add(edgeNode);
}
}
return filteredEdgeNodes;
}
/*TODO:
* Add Logging info
*/
}
/**
* Adding BYON nodes to a job component
* @param byonIdPerComponent a mapping between byon nodes and job components
......@@ -763,6 +892,59 @@ public class PAGateway {
return 0;
}
/**
* Adding EDGE nodes to a job component
* @param edgeIdPerComponent a mapping between byon nodes and job components
* @param jobId A constructed job identifier
* @return 0 if nodes has been added properly. A greater than 0 value otherwise.
*/
public int addEdgeNodes(Map<String, String> edgeIdPerComponent, String jobId) {
Validate.notNull(edgeIdPerComponent, "The received byonIdPerComponent structure is empty. Nothing to be added.");
EntityManagerHelper.begin();
edgeIdPerComponent.forEach((edgeNodeId, componentName) -> {
EdgeNode edgeNode = EntityManagerHelper.find(EdgeNode.class, edgeNodeId);
Task task = EntityManagerHelper.find(Task.class, jobId + componentName);
assert edgeNode!=null : "The EDGE ID passed in the mapping does not exist in the database";
assert task!=null : "The componentId passed in the mapping does not exist in the database";
Deployment newDeployment = new Deployment();
newDeployment.setNodeName(edgeNode.getName());
newDeployment.setDeploymentType(NodeType.EDGE);
newDeployment.setEdgeNode(edgeNode);
PACloud cloud = new PACloud();
String nodeSourceName = "EDGE_NS_" + edgeNode.getId();
cloud.setCloudID(nodeSourceName);
cloud.setNodeSourceNamePrefix(nodeSourceName);
cloud.setCloudType(CloudType.EDGE);
cloud.addDeployment(newDeployment);
newDeployment.setPaCloud(cloud);
EntityManagerHelper.persist(cloud);
List<EdgeNode> edgeNodeList = new LinkedList<>();
edgeNodeList.add(edgeNode);
LOGGER.info("EDGE node Added: " + edgeNode.getName() + " Ip: " + edgeNode.getIpAddresses().get(0).getValue());
defineEdgeNodeSource(edgeNodeList, nodeSourceName);
LOGGER.info("EDGE node source EDGE_NS_" + edgeNode.getId() + " is defined");
newDeployment.setTask(task);
newDeployment.setNumber(task.getNextDeploymentID());
EntityManagerHelper.persist(newDeployment);
LOGGER.debug("Deployment created: " + newDeployment.toString());
task.addDeployment(newDeployment);
EntityManagerHelper.persist(task);
});
EntityManagerHelper.commit();
LOGGER.info("EDGE nodes added properly.");
return 0;
}
/**
* Undeploy clouds
* @param cloudIDs List of cloud IDs to remove
......@@ -1127,6 +1309,7 @@ public class PAGateway {
null,
null,
NodeType.IAAS,
null,
null
);
})
......
scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ByonUtils.java
View file @ 20da4ad1
......@@ -28,75 +28,96 @@ public class ByonUtils {
}
/**
* @param byonDef an Object of class ByonDefinition that contains all the nodes Definition
* @param np an Object of class NodeProperties that contains all the nodes properties needed for the candidate declaration
* @param jobId a String identifier of the node candidate job
* @param nodeType a String of the node type (byon or edge)
* @return an object of class NodeCandidate
*/
public static NodeCandidate createByonNodeCandidate(ByonDefinition byonDef, String jobId) {
LOGGER.debug("Creating the BYON node candidate ...");
//create a dummy cloud
Cloud byonCloud = ByonUtils.getOrCreateDummyByonCloud();
public static NodeCandidate createNodeCandidate(NodeProperties np, String jobId, String nodeType) {
EntityManagerHelper.begin();
//set the image
Image image = new Image();
image.setId("byon-image-" + RandomStringUtils.randomAlphanumeric(16));
image.setOperatingSystem(byonDef.getNodeProperties().getOperatingSystem());
LOGGER.debug("Creating the {} node candidate ...", nodeType.toUpperCase());
//Start by setting the universal nodes properties
NodeCandidate nc = new NodeCandidate();
nc.setJobIdForBYON(jobId);
nc.setPrice(0.0);
nc.setMemoryPrice(0.0);
nc.setPricePerInvocation(0.0);
//set the hardware
//create a dummy cloud definition for BYON nodes
Cloud dummyCloud = new Cloud();
dummyCloud = ByonUtils.getOrCreateDummyCloud(nodeType);
//Create a dummy image
Image image = new Image();
image.setOperatingSystem(np.getOperatingSystem());
//Define the hardware
Hardware hardware = new Hardware();
hardware.setId("byon-hardware-" + RandomStringUtils.randomAlphanumeric(16));
hardware.setCores(byonDef.getNodeProperties().getNumberOfCores());
hardware.setDisk((double) byonDef.getNodeProperties().getDisk());
hardware.setRam(byonDef.getNodeProperties().getMemory());
//set the location
hardware.setCores(np.getNumberOfCores());
hardware.setDisk((double) np.getDisk());
hardware.setRam(np.getMemory());
//Define the location
Location location = new Location();
location.setId("byon-location-" + RandomStringUtils.randomAlphanumeric(16));
location.setGeoLocation(byonDef.getNodeProperties().getGeoLocation());
//define a dummy NC
NodeCandidate byonNC = new NodeCandidate();
byonNC.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.BYON);
byonNC.setJobIdForBYON(jobId);
byonNC.setPrice(0.0);
byonNC.setMemoryPrice(0.0);
byonNC.setPricePerInvocation(0.0);
byonNC.setCloud(byonCloud);
byonNC.setImage(image);
byonNC.setHardware(hardware);
byonNC.setLocation(location);
EntityManagerHelper.persist(byonNC);
EntityManagerHelper.commit();
LOGGER.info("BYON node candidate created.");
return byonNC;
location.setGeoLocation(np.getGeoLocation());
//Define the properties that depend on the node type
if (nodeType.equals("byon")) {
//set the image name
image.setId("byon-image-" + RandomStringUtils.randomAlphanumeric(16));
//set the hardware
hardware.setId("byon-hardware-" + RandomStringUtils.randomAlphanumeric(16));
//set the location
location.setId("byon-location-" + RandomStringUtils.randomAlphanumeric(16));
//set the nc parameters
nc.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.BYON);
}
else { //the node type is EDGE
//set the image name
image.setId("edge-image-" + RandomStringUtils.randomAlphanumeric(16));
//set the hardware
hardware.setId("edge-hardware-" + RandomStringUtils.randomAlphanumeric(16));
//set the location
location.setId("edge-location-" + RandomStringUtils.randomAlphanumeric(16));
//set the nc parameters
nc.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.EDGE);
}
nc.setCloud(dummyCloud);
nc.setImage(image);
nc.setHardware(hardware);
nc.setLocation(location);
EntityManagerHelper.persist(nc);
LOGGER.info("{} node candidate created.", nodeType.toUpperCase());
return nc;
}
/**
* Create a dummy object of class Cloud to be used for the node candidates
* @return the created byonCloud object
*/
public static Cloud getOrCreateDummyByonCloud() {
LOGGER.debug("Searching for the dummy BYON cloud ...");
public static Cloud getOrCreateDummyCloud(String nodeType) {
LOGGER.debug("Searching for the dummy cloud ...");
EntityManagerHelper.begin();
//Check if the Byon cloud already exists
Optional<Cloud> optCloud = Optional.ofNullable(EntityManagerHelper.find(Cloud.class, "byon"));
Optional<Cloud> optCloud = Optional.ofNullable(EntityManagerHelper.find(Cloud.class, nodeType));
if (optCloud.isPresent()) {
LOGGER.info("Dummy BYON cloud found!");
LOGGER.info("Dummy cloud for {} was found!", nodeType);
return optCloud.get();
}
LOGGER.debug("Creating the dummy BYON cloud ...");
LOGGER.debug("Creating the dummy cloud for {} Nodes ...", nodeType);
//else, Byon cloud will be created
Cloud byonCloud = new Cloud();
byonCloud.setCloudType(CloudType.BYON);
byonCloud.setOwner("BYON");
byonCloud.setId("byon");
Cloud newCloud = new Cloud();
newCloud.setCloudType((nodeType.equals("byon")) ? CloudType.BYON : CloudType.EDGE);
newCloud.setOwner((nodeType.equals("byon")) ? "BYON" : "EDGE");
newCloud.setId(nodeType);
//Add the Byon cloud to the database
EntityManagerHelper.persist(byonCloud);
EntityManagerHelper.commit();
LOGGER.info("Dummy BYON cloud created.");
return byonCloud;
EntityManagerHelper.persist(newCloud);
LOGGER.info("Dummy {} cloud created.", nodeType.toUpperCase());
return newCloud;
/* TODO :
* Check if we have to add other variables to the new cloud
......
scheduling-abstraction-layer/src/main/resources/Define_NS_BYON.xml
View file @ 20da4ad1
<?xml version="1.0" encoding="UTF-8"?>
<job
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="urn:proactive:jobdescriptor:3.12" xsi:schemaLocation="urn:proactive:jobdescriptor:3.12 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.12/schedulerjob.xsd" name="Define_NS_BYON.xml" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" >
xmlns="urn:proactive:jobdescriptor:3.12" xsi:schemaLocation="urn:proactive:jobdescriptor:3.12 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.12/schedulerjob.xsd" name="Define_NS_BYON" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" >
<variables>
<variable name="pa_port" value="8080" model="PA:Integer"/>
<variable name="NS_name" value="SSH_INFRA" model="PA:NOT_EMPTY_STRING"/>
......@@ -75,7 +75,11 @@ def infrastructureParameters = ["300000", //Node Time out
"/opt/ProActive_node_agent/jre/bin/java", //JavaPath on the remote host
"/opt/ProActive_node_agent", //ScheduligPath on the remote hosts
"Linux", //targetOs
javaOptions]//Java options
javaOptions, //Java options
"autoGenerated", //deploymentMode
"", //nodeJarUrl
"", //startupScriptStandard
""] //startupScriptWithNodeJarDownload
def infrastructureFileParameters = [hosts, //hostsList file content
"", //SSH Private Key
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment