diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f29f9c80197a4eaa415906ff9c86b8374688fd5e..a446d1e7ecd59dd9775ab2419b1afc0d5e2fa2ac 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -146,6 +146,7 @@ deploy:scheduling-abstraction-layer:
- morphemic-rc1.5
- proactive-dev
- fix-byon-acquire
+ - fix-162
dependencies:
- build:scheduling-abstraction-layer
script:
diff --git a/scheduling-abstraction-layer/pom.xml b/scheduling-abstraction-layer/pom.xml
index d98a0e9cb5bae773852f83a72335e8dd8575048e..88a80d233773191cd9fe596027c1b6d83da5e70a 100644
--- a/scheduling-abstraction-layer/pom.xml
+++ b/scheduling-abstraction-layer/pom.xml
@@ -4,7 +4,7 @@
4.0.0
org.activeeon
scheduling-abstraction-layer
- 3.7-SNAPSHOT
+ 4.0-SNAPSHOT
org.projectlombok
@@ -68,6 +68,11 @@
5.4.22.Final
compile
+
+ org.javatuples
+ javatuples
+ 1.2
+
org.hibernate
hibernate-core
diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java
index a5bd99e11826f38cefca647e49d2221c12701bc5..f4c5521491b193c0efce29309e10d7e5e72c4da4 100644
--- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java
+++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/infrastructure/deployment/PAConnectorIaasGateway.java
@@ -3,15 +3,13 @@ package org.activeeon.morphemic.infrastructure.deployment;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.activeeon.morphemic.model.PACloud;
+import org.activeeon.morphemic.service.ConnectionHelper;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.Validate;
import org.apache.http.client.utils.URIBuilder;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.json.JSONArray;
-import org.json.JSONTokener;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
+import org.json.JSONObject;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
@@ -33,35 +31,22 @@ public class PAConnectorIaasGateway {
}
@SneakyThrows
- public JSONArray getNodeCandidates(String nodeSourceName, String region, String imageReq) {
+ public JSONObject getNodeCandidates(String nodeSourceName, String region, String imageReq, String token) {
Validate.notNull(nodeSourceName, "nodeSourceName must not be null");
Validate.notNull(region, "region must not be null");
LOGGER.info("Retrieving node candidates for cloud " + nodeSourceName + " region " +
region + " and imageReq " + imageReq);
- JSONArray nodeCandidates = null;
+ JSONObject nodeCandidates;
URIBuilder uriBuilder = new URIBuilder(new URL(paURL).toURI());
URI requestUri = uriBuilder.setPath(CONNECTOR_IAAS_PATH + "/infrastructures/" + nodeSourceName + "/nodecandidates")
.addParameter("region", region)
.addParameter("imageReq", imageReq)
+ .addParameter("nextToken", token)
.build();
- HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
- connection.setRequestMethod(HttpMethod.POST.toString());
- LOGGER.info("requestUri = " + requestUri.toString());
-
- if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- LOGGER.error("Failed : HTTP error code : " + connection.getResponseCode());
- return null;
- }
-
- BufferedReader br = new BufferedReader(new InputStreamReader(
- (connection.getInputStream())));
-
- nodeCandidates = new JSONArray(new JSONTokener(br));
- LOGGER.info("Node candidates retrieved for successfully: " + nodeCandidates.toString());
-
- connection.disconnect();
+ nodeCandidates = ConnectionHelper.sendGetRequestAndReturnObjectResponse(requestUri);
+ LOGGER.info("Node candidates retrieved for successfully: {}", nodeCandidates);
return nodeCandidates;
}
@@ -76,22 +61,8 @@ public class PAConnectorIaasGateway {
URI requestUri = uriBuilder.setPath(CONNECTOR_IAAS_PATH + "/infrastructures/" + nodeSourceName + "/images")
.build();
- HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
- connection.setRequestMethod(HttpMethod.GET.toString());
- LOGGER.info("requestUri = " + requestUri.toString());
-
- if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- LOGGER.error("Failed : HTTP error code : " + connection.getResponseCode());
- return null;
- }
-
- BufferedReader br = new BufferedReader(new InputStreamReader(
- (connection.getInputStream())));
-
- images = new JSONArray(new JSONTokener(br));
- LOGGER.info("Images retrieved for cloud " + nodeSourceName + ". Images: " + images.toString());
-
- connection.disconnect();
+ images = ConnectionHelper.sendGetArrayRequestAndReturnArrayResponse(requestUri);
+ LOGGER.info("Images retrieved for cloud {}. Images: {}", nodeSourceName, images);
return images;
}
@@ -106,22 +77,8 @@ public class PAConnectorIaasGateway {
URI requestUri = uriBuilder.setPath(CONNECTOR_IAAS_PATH + "/infrastructures/" + nodeSourceName + "/regions")
.build();
- HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
- connection.setRequestMethod(HttpMethod.GET.toString());
- LOGGER.debug("requestUri = " + requestUri.toString());
-
- if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- LOGGER.error("Failed : HTTP error code : " + connection.getResponseCode());
- return null;
- }
-
- BufferedReader br = new BufferedReader(new InputStreamReader(
- (connection.getInputStream())));
-
- regions = new JSONArray(new JSONTokener(br));
- LOGGER.debug("Regions retrieved for cloud " + nodeSourceName + ". Images: " + regions.toString());
-
- connection.disconnect();
+ regions = ConnectionHelper.sendGetArrayRequestAndReturnArrayResponse(requestUri);
+ LOGGER.debug("Regions retrieved for cloud {}. Images: {}", nodeSourceName, regions);
return regions;
}
@@ -165,10 +122,10 @@ public class PAConnectorIaasGateway {
os.write(input, 0, input.length);
}
- LOGGER.debug("requestUri = " + requestUri.toString());
+ LOGGER.debug("requestUri = {}", requestUri);
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- LOGGER.error("Failed : HTTP error code : " + connection.getResponseCode());
+ LOGGER.error("Failed : HTTP error code : {}", connection.getResponseCode());
}
LOGGER.debug("Infrastructure defined successfully.");
@@ -219,10 +176,10 @@ public class PAConnectorIaasGateway {
HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
connection.setRequestMethod(HttpMethod.DELETE.toString());
- LOGGER.debug("requestUri = " + requestUri.toString());
+ LOGGER.debug("requestUri = {}", requestUri);
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
- LOGGER.error("Failed : HTTP error code : " + connection.getResponseCode());
+ LOGGER.error("Failed : HTTP error code : {}", connection.getResponseCode());
}
LOGGER.debug("Infrastructure deleted successfully.");
diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/nc/NodeCandidateUtils.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/nc/NodeCandidateUtils.java
index 0086c93e37c4552aee7f653ab502d27ad7f1b3f1..3d26d19c274944a605830749479396670c4b21f1 100644
--- a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/nc/NodeCandidateUtils.java
+++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/nc/NodeCandidateUtils.java
@@ -1,12 +1,16 @@
package org.activeeon.morphemic.nc;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import lombok.extern.slf4j.Slf4j;
import org.activeeon.morphemic.infrastructure.deployment.PAConnectorIaasGateway;
import org.activeeon.morphemic.model.*;
import org.activeeon.morphemic.service.EntityManagerHelper;
import org.activeeon.morphemic.service.GeoLocationUtils;
import org.apache.commons.lang3.StringUtils;
+import org.javatuples.Quartet;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -14,7 +18,10 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
@Slf4j
public class NodeCandidateUtils {
@@ -23,9 +30,23 @@ public class NodeCandidateUtils {
private final GeoLocationUtils geoLocationUtils;
+ private final LoadingCache, JSONArray> nodeCandidatesCache;
+
public NodeCandidateUtils(String paURL) {
connectorIaasGateway = new PAConnectorIaasGateway(paURL);
geoLocationUtils = new GeoLocationUtils();
+ nodeCandidatesCache = CacheBuilder.newBuilder()
+ .maximumSize(100)
+ .expireAfterWrite(60, TimeUnit.MINUTES)
+ .build(new CacheLoader, JSONArray>() {
+ @Override
+ public JSONArray load(Quartet key) {
+ return getAllPagedNodeCandidates(key.getValue0(),
+ key.getValue1(),
+ key.getValue2(),
+ key.getValue3());
+ }
+ });
}
public static boolean verifyAllFilters(List requirements, NodeCandidate nodeCandidate) {
@@ -135,7 +156,6 @@ public class NodeCandidateUtils {
}
private Location createLocation(JSONObject nodeCandidateJSON, PACloud paCloud) {
- LOGGER.debug("Creating location ...");
String locationId = paCloud.getCloudID() + "/" + nodeCandidateJSON.optString("region");
Location location = EntityManagerHelper.find(Location.class, locationId);
if (location == null) {
@@ -147,7 +167,6 @@ public class NodeCandidateUtils {
location.setIsAssignable(true);
location.setGeoLocation(createGeoLocation(paCloud.getCloudProviderName(), location.getName()));
}
- LOGGER.debug("Location created: " + location.toString());
return location;
}
@@ -210,7 +229,6 @@ public class NodeCandidateUtils {
}
public NodeCandidate createNodeCandidate(JSONObject nodeCandidateJSON, JSONObject imageJSON, PACloud paCloud) {
- LOGGER.debug("Creating node candidate ...");
NodeCandidate nodeCandidate = new NodeCandidate();
nodeCandidate.setNodeCandidateType(NodeCandidate.NodeCandidateTypeEnum.IAAS);
nodeCandidate.setPrice(nodeCandidateJSON.optDouble("price"));
@@ -223,7 +241,6 @@ public class NodeCandidateUtils {
nodeCandidate.setPricePerInvocation((double) 0);
nodeCandidate.setMemoryPrice((double) 0);
nodeCandidate.setEnvironment(new Environment());
- LOGGER.debug("Node candidate created: " + nodeCandidate.toString());
return nodeCandidate;
}
@@ -281,31 +298,57 @@ public class NodeCandidateUtils {
throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudProviderName() + " is not handled yet.");
}
- if (paCloud.getCloudProviderName().equals("openstack")){
- if((!entries.contains(pair)) && openstackOsList.contains(os)) {
- entries.add(pair);
- JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(),
- region, imageReq);
+ try {
+ if (paCloud.getCloudProviderName().equals("openstack")){
+ if((!entries.contains(pair)) && openstackOsList.contains(os)) {
+ entries.add(pair);
+ JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
+ nodeCandidates.forEach(nc -> {
+ JSONObject nodeCandidate = (JSONObject) nc;
+ EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud));
+ EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud));
+ });
+
+ }
+ }
+ else {
+ JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
nodeCandidates.forEach(nc -> {
JSONObject nodeCandidate = (JSONObject) nc;
EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud));
EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud));
});
-
}
+ } catch (ExecutionException ee) {
+ LOGGER.error("Could not get node candidates from cache: ", ee);
}
- else {
- JSONArray nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(),
- region, imageReq);
- nodeCandidates.forEach(nc -> {
- JSONObject nodeCandidate = (JSONObject) nc;
- EntityManagerHelper.persist(createLocation(nodeCandidate, paCloud));
- EntityManagerHelper.persist(createNodeCandidate(nodeCandidate, image, paCloud));
- });
- }
+
});
});
EntityManagerHelper.commit();
}
+
+ private JSONArray getAllPagedNodeCandidates(PACloud paCloud, String region, String imageReq, String token) {
+ JSONObject nodeCandidates = connectorIaasGateway.getNodeCandidates(paCloud.getDummyInfrastructureName(),
+ region, imageReq, token);
+ try {
+ if (!nodeCandidates.optString("nextToken").isEmpty()) {
+ return (joinJSONArrays(nodeCandidates.optJSONArray("nodeCandidates"),
+ nodeCandidatesCache.get(Quartet.with(paCloud,
+ region,
+ imageReq,
+ nodeCandidates.optString("nextToken")))));
+ }
+ } catch (ExecutionException ee) {
+ LOGGER.error("Could not get node candidates from cache: ", ee);
+ }
+ return (nodeCandidates.optJSONArray("nodeCandidates"));
+ }
+
+ private JSONArray joinJSONArrays(JSONArray... jsonArrays) {
+ JSONArray resultJSONArray = new JSONArray();
+ Arrays.stream(jsonArrays).forEach(jsonArray -> IntStream.range(0, jsonArray.length()).mapToObj(jsonArray::get).forEach(resultJSONArray::put));
+ return resultJSONArray;
+ }
}
diff --git a/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ConnectionHelper.java b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ConnectionHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..2303076c24ed407cf05543d3b62021998afe0c1c
--- /dev/null
+++ b/scheduling-abstraction-layer/src/main/java/org/activeeon/morphemic/service/ConnectionHelper.java
@@ -0,0 +1,59 @@
+package org.activeeon.morphemic.service;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.jboss.netty.handler.codec.http.HttpMethod;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+
+@Slf4j
+public class ConnectionHelper {
+
+ private ConnectionHelper() {}
+
+ @SneakyThrows
+ private static BufferedReader sendGetRequestAndReturnBufferedResponse (HttpURLConnection connection) {
+ if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+ LOGGER.error("Failed : HTTP error code : {}", connection.getResponseCode());
+ return null;
+ }
+
+ return new BufferedReader(new InputStreamReader((connection.getInputStream())));
+ }
+
+ @SneakyThrows
+ public static JSONObject sendGetRequestAndReturnObjectResponse(URI requestUri) {
+ HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
+ connection.setRequestMethod(HttpMethod.GET.toString());
+ LOGGER.debug("requestUri = {}", requestUri);
+
+ BufferedReader br = sendGetRequestAndReturnBufferedResponse(connection);
+
+ JSONObject result = (br != null) ? new JSONObject (new JSONTokener(br)) : null;
+
+ connection.disconnect();
+
+ return result;
+ }
+
+ @SneakyThrows
+ public static JSONArray sendGetArrayRequestAndReturnArrayResponse(URI requestUri) {
+ HttpURLConnection connection = (HttpURLConnection) requestUri.toURL().openConnection();
+ connection.setRequestMethod(HttpMethod.GET.toString());
+ LOGGER.debug("requestUri = {}", requestUri);
+
+ BufferedReader br = sendGetRequestAndReturnBufferedResponse(connection);
+
+ JSONArray result = (br != null) ? new JSONArray (new JSONTokener(br)) : null;
+
+ connection.disconnect();
+
+ return result;
+ }
+}