From 58175ab3291888f72c210eaefe15a3a51cb85925 Mon Sep 17 00:00:00 2001 From: mklkun Date: Thu, 8 Oct 2020 11:21:21 +0200 Subject: [PATCH 1/2] Add lazy-mode jobs creation and submission --- build.gradle | 1 + .../org/activeeon/morphemic/PAGateway.java | 182 ++++++++++++++++++ .../application/deployment/PAFactory.java | 25 ++- .../deployment/PAResourceManagerGateway.java | 6 +- .../org/activeeon/morphemic/model/Job.java | 7 +- .../org/activeeon/morphemic/model/Task.java | 29 ++- 6 files changed, 232 insertions(+), 18 deletions(-) create mode 100644 src/main/java/org/activeeon/morphemic/PAGateway.java diff --git a/build.gradle b/build.gradle index 9ba5cefc..56104049 100644 --- a/build.gradle +++ b/build.gradle @@ -39,6 +39,7 @@ dependencies { compile group: 'org.hibernate', name: 'hibernate-core', version: '5.4.22.Final' compile group: 'org.projectlombok', name: 'lombok', version: '1.18.12' compile group: 'org.hsqldb', name: 'hsqldb', version: '2.5.1' + compile group: 'org.json', name: 'json', version: '20200518' annotationProcessor "org.projectlombok:lombok:1.18.12" } diff --git a/src/main/java/org/activeeon/morphemic/PAGateway.java b/src/main/java/org/activeeon/morphemic/PAGateway.java new file mode 100644 index 00000000..f8fb376f --- /dev/null +++ b/src/main/java/org/activeeon/morphemic/PAGateway.java @@ -0,0 +1,182 @@ +package org.activeeon.morphemic; + +import org.activeeon.morphemic.application.deployment.PAFactory; +import org.activeeon.morphemic.application.deployment.PASchedulerGateway; +import org.activeeon.morphemic.infrastructure.deployment.PAResourceManagerGateway; +import org.activeeon.morphemic.model.Job; +import org.activeeon.morphemic.model.Task; +import org.apache.log4j.Logger; +import org.json.JSONObject; +import org.ow2.proactive.resourcemanager.exception.RMException; +import org.ow2.proactive.scheduler.common.exception.UserException; +import org.ow2.proactive.scheduler.common.job.TaskFlowJob; +import org.ow2.proactive.scheduler.common.task.ScriptTask; + +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.Persistence; +import javax.security.auth.login.LoginException; +import java.security.KeyException; +import java.util.LinkedList; +import java.util.List; + +public class PAGateway { + + public PAResourceManagerGateway resourceManagerGateway; + + public PASchedulerGateway schedulerGateway; + + private final EntityManager em; + + private static final Logger LOGGER = Logger.getLogger(PAGateway.class); + + /** + * Construct a gateway to the ProActive server + * @param paRestURL ProActive server rest URL (exp: http://try.activeeon.com:8080/rest) + */ + public PAGateway(String paRestURL) { + resourceManagerGateway = new PAResourceManagerGateway(paRestURL); + schedulerGateway = new PASchedulerGateway(paRestURL); + + EntityManagerFactory emf = Persistence.createEntityManagerFactory("model"); + em = emf.createEntityManager(); + } + + /** + * Connect to the ProActive server + * @param username The user's username + * @param password The user's password + * @throws LoginException In case the login is not valid + * @throws KeyException In case the password is not valid + * @throws RMException In case an error happens in the RM + */ + public void connect(String username, String password) throws LoginException, KeyException, RMException { + LOGGER.debug("Connecting to ProActive's Resource Manager"); + resourceManagerGateway.connect(username, password); + LOGGER.debug("Connecting to ProActive's Scheduler"); + schedulerGateway.connect(username,password); + } + + /** + * Disconnect from the ProActive server + */ + public void disconnect() { + LOGGER.debug("Disconnecting from RM..."); + resourceManagerGateway.disconnect(); + LOGGER.info("Disconnected from RM."); + LOGGER.debug("Disconnecting from Scheduler..."); + schedulerGateway.disconnect(); + LOGGER.info("Disconnected from Scheduler."); + } + + /** + * Create a ProActive job skeleton + * @param job A job skeleton definition in JSON format + */ + public void createJob(JSONObject job) { + if (job==null){ + LOGGER.warn("The job received is empty. Nothing to be created"); + return; + } + + em.getTransaction().begin(); + + Job newJob = new Job(); + newJob.setJobId(job.optJSONObject("jobInformation").optString("id")); + newJob.setName(job.optJSONObject("jobInformation").optString("name")); + List tasks = new LinkedList<>(); + JSONObject jsonTasks = job.optJSONObject("tasks"); + jsonTasks.keySet().forEach(key -> { + JSONObject value = jsonTasks.optJSONObject(key); + Task newTask = new Task(); + newTask.setTaskId(value.optString("id")); + newTask.setName(key); + newTask.setType(value.optString("type")); + newTask.setPreInstall(value.optString("preInstall")); + newTask.setInstall(value.optString("install")); + newTask.setPostInstall(value.optString("postInstall")); + newTask.setPreStart(value.optString("preStart")); + newTask.setStart(value.optString("start")); + newTask.setPostStart(value.optString("postStart")); + newTask.setPreStop(value.optString("preStop")); + newTask.setStop(value.optString("stop")); + newTask.setPostStop(value.optString("postStop")); + + em.persist(newTask); + tasks.add(newTask); + }); + + newJob.setTasks(tasks); + + em.persist(newJob); + + em.getTransaction().commit(); + + LOGGER.info("Job created: " + newJob.toString()); + } + + /** + * Translate a Morphemic task skeleton into a list of ProActive tasks + * @param task A Morphemic task skeleton + * @return A list of ProActive tasks + */ + public List buildPATask(Task task) { + List scriptTasks = new LinkedList<>(); + if ("bash".equals(task.getType())) { + if (!task.getInstall().isEmpty()) { + ScriptTask scriptTaskInstall = PAFactory.createBashScriptTask(task.getName() + "_install", task.getInstall()); + if (!task.getPreInstall().isEmpty()) { + scriptTaskInstall.setPreScript(PAFactory.createSimpleScript(task.getPreInstall(), "bash")); + } + if (!task.getPostInstall().isEmpty()) { + scriptTaskInstall.setPostScript(PAFactory.createSimpleScript(task.getPostInstall(), "bash")); + } + scriptTasks.add(scriptTaskInstall); + } + + if (!task.getStart().isEmpty()) { + ScriptTask scriptTaskStart = PAFactory.createBashScriptTask(task.getName() + "_start", task.getStart()); + if (!task.getPreStart().isEmpty()) { + scriptTaskStart.setPreScript(PAFactory.createSimpleScript(task.getPreStart(), "bash")); + } + if (!task.getPostStart().isEmpty()) { + scriptTaskStart.setPostScript(PAFactory.createSimpleScript(task.getPostStart(), "bash")); + } + scriptTasks.add(scriptTaskStart); + } + } + + return scriptTasks; + } + + /** + * Submit a job constructed in lazy-mode to the ProActive Scheduler + * @param jobId A constructed job identifier + * @return The submitted job id + */ + public long submitJob(String jobId) { + Job jobToSubmit = em.find(Job.class, jobId); + LOGGER.info("Job found to submit: " + jobToSubmit.toString()); + + TaskFlowJob paJob = new TaskFlowJob(); + paJob.setName(jobToSubmit.getName()); + + //TODO + //paJob.setVariables(PAFactory.variablesToJobVariables(jobToSubmit.getVariables())); + + LOGGER.info("Job created: " + paJob.toString()); + + jobToSubmit.getTasks().forEach(task -> { + List scriptTasks = buildPATask(task); + scriptTasks.forEach(scriptTask -> { + try { + paJob.addTask(scriptTask); + } catch (UserException e) { + LOGGER.error("Task " + task.getName() + " could not be added due to: " + e.toString()); + } + }); + }); + + return(schedulerGateway.submit(paJob).longValue()); + } +} diff --git a/src/main/java/org/activeeon/morphemic/application/deployment/PAFactory.java b/src/main/java/org/activeeon/morphemic/application/deployment/PAFactory.java index 7827b386..3e69d471 100644 --- a/src/main/java/org/activeeon/morphemic/application/deployment/PAFactory.java +++ b/src/main/java/org/activeeon/morphemic/application/deployment/PAFactory.java @@ -61,13 +61,36 @@ public class PAFactory { return mySQLSimpleScript; } + /** + * Create a Bash script task + * @param taskName The name of the task + * @param implementationScript The script implementation + * @return A ProActive ScriptTask instance + */ + public static ScriptTask createBashScriptTask(String taskName, String implementationScript) { + ScriptTask mySQLTask = new ScriptTask(); + mySQLTask.setName(taskName); + SimpleScript mySQLSimpleScript; + TaskScript mySQLTaskScript = null; + LOGGER.debug("Creating a bash script task"); + try { + mySQLSimpleScript = createSimpleScript(implementationScript, "bash"); + mySQLTaskScript = new TaskScript(mySQLSimpleScript); + } catch (InvalidScriptException ie) { + LOGGER.error("ERROR: Task " + taskName + " script not created due to an InvalidScriptException: " + ie.toString()); + } + LOGGER.debug("Bash script task created."); + mySQLTask.setScript(mySQLTaskScript); + return mySQLTask; + } + /** * Create a Bash script task * @param taskName The name of the task * @param scriptFileName The script implementation file name * @return A ProActive ScriptTask instance */ - public static ScriptTask createBashScriptTask(String taskName, String scriptFileName) { + public static ScriptTask createBashScriptTaskFromFile(String taskName, String scriptFileName) { ScriptTask mySQLTask = new ScriptTask(); mySQLTask.setName(taskName); SimpleScript mySQLSimpleScript; diff --git a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java index 69b84adb..710d0c5b 100644 --- a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java +++ b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java @@ -166,7 +166,11 @@ public class PAResourceManagerGateway { "http://sd-79787.dedibox.fr:8080/connector-iaas", "http://sd-79787.dedibox.fr:8080/rest/node.jar", "", - "300000"}; + "300000", + "mkdir -p /tmp/node && cd /tmp/node\n" + + " if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u131-linux-x64.tar.gz; tar -xf jre-8u131-linux-x64.tar.gz; mv jre1.8.0_131/ jre; fi\n" + + "wget -nv %nodeJarUrl%\n" + + "nohup jre/bin/java -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &"}; LOGGER.debug("infrastructureParameters: " + Arrays.toString(infrastructureParameters)); String[] infrastructureFileParameters = {""}; String policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.StaticPolicy"; diff --git a/src/main/java/org/activeeon/morphemic/model/Job.java b/src/main/java/org/activeeon/morphemic/model/Job.java index 3d611cd4..d99e152a 100644 --- a/src/main/java/org/activeeon/morphemic/model/Job.java +++ b/src/main/java/org/activeeon/morphemic/model/Job.java @@ -5,6 +5,7 @@ import lombok.*; import javax.persistence.*; import java.io.Serializable; import java.util.List; +import java.util.Map; @AllArgsConstructor @@ -19,11 +20,15 @@ public class Job implements Serializable { // Uncomment this for an automatically generated id //@GeneratedValue(strategy = GenerationType.AUTO) @Column(name = "JOB_ID") - private long jobId; + private String jobId; @Column(name = "NAME") private String name; + @Column(name = "VARIABLES") + @ElementCollection(targetClass=String.class) + private Map variables; + @OneToMany(fetch = FetchType.EAGER, orphanRemoval = true) private List tasks; } diff --git a/src/main/java/org/activeeon/morphemic/model/Task.java b/src/main/java/org/activeeon/morphemic/model/Task.java index 805b8b22..4fbeacf2 100644 --- a/src/main/java/org/activeeon/morphemic/model/Task.java +++ b/src/main/java/org/activeeon/morphemic/model/Task.java @@ -1,9 +1,6 @@ package org.activeeon.morphemic.model; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; +import lombok.*; import javax.persistence.*; import java.io.Serializable; @@ -13,13 +10,15 @@ import java.io.Serializable; @NoArgsConstructor @ToString @Getter +@Setter @Entity @Table(name = "TASK") public class Task implements Serializable { @Id - @GeneratedValue(strategy = GenerationType.AUTO) + // Uncomment this for an automatically generated id + //@GeneratedValue(strategy = GenerationType.AUTO) @Column(name = "TASK_ID") - private long taskId; + private String taskId; @Column(name = "NAME") private String name; @@ -27,30 +26,30 @@ public class Task implements Serializable { @Column(name = "TYPE") private String type; - @Column(name = "PREINSTALL") + @Column(name = "PREINSTALL", length = 5000) private String preInstall; - @Column(name = "INSTALL") + @Column(name = "INSTALL", length = 5000) private String install; - @Column(name = "POSTINSTALL") + @Column(name = "POSTINSTALL", length = 5000) private String postInstall; - @Column(name = "PRESTART") + @Column(name = "PRESTART", length = 5000) private String preStart; - @Column(name = "START") + @Column(name = "START", length = 5000) private String start; - @Column(name = "POSTSTART") + @Column(name = "POSTSTART", length = 5000) private String postStart; - @Column(name = "PRESTOP") + @Column(name = "PRESTOP", length = 5000) private String preStop; - @Column(name = "STOP") + @Column(name = "STOP", length = 5000) private String stop; - @Column(name = "POSTSTOP") + @Column(name = "POSTSTOP", length = 5000) private String postStop; } -- GitLab From 0b2060697c5ec58e3d3c2d737c92d1c7d90a6182 Mon Sep 17 00:00:00 2001 From: mklkun Date: Thu, 8 Oct 2020 11:28:54 +0200 Subject: [PATCH 2/2] Revert infrastructure info --- .../infrastructure/deployment/PAResourceManagerGateway.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java index 710d0c5b..69b84adb 100644 --- a/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java +++ b/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAResourceManagerGateway.java @@ -166,11 +166,7 @@ public class PAResourceManagerGateway { "http://sd-79787.dedibox.fr:8080/connector-iaas", "http://sd-79787.dedibox.fr:8080/rest/node.jar", "", - "300000", - "mkdir -p /tmp/node && cd /tmp/node\n" + - " if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u131-linux-x64.tar.gz; tar -xf jre-8u131-linux-x64.tar.gz; mv jre1.8.0_131/ jre; fi\n" + - "wget -nv %nodeJarUrl%\n" + - "nohup jre/bin/java -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &"}; + "300000"}; LOGGER.debug("infrastructureParameters: " + Arrays.toString(infrastructureParameters)); String[] infrastructureFileParameters = {""}; String policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.StaticPolicy"; -- GitLab