Update .gitlab-ci.yml

7718a8c7 · Anna Warno · 3ad9c26d · 7718a8c7
Commit 7718a8c7 authored Mar 28, 2022 by Anna Warno
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,9 +11,13 @@ variables:
  DOCKER_DRIVER: overlay
  DOCKER_TLS_CERTDIR: "/certs"

+  DOCKER_CLI: "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $CI_PROJECT_DIR/maven_repo:/root/.m2 -w /usr/src/mymaven -v $CI_PROJECT_DIR:/usr/src/mymaven $MAVEN_IMAGE"
  SCHEDULING_ABSTRACTION_LAYER_CLI: "mvn --batch-mode -f scheduling-abstraction-layer/pom.xml"
  AMQ_MESSAGE_JAVA_LIBRARY_CLI: "mvn --batch-mode -f amq-message-java-library/pom.xml"
  SLO_SEVERITY_CALCULATOR_CLI: "mvn --batch-mode -f morphemic-slo-severity-calculator/pom.xml"
+  PREDICTON_ORCHESTRATOR_CLI: "mvn --batch-mode -N -f prediction_orchestrator/pom.xml"
+
+  DOCKER_FORECAST_ESHYBRID_DOCKER_NAME: 'morphemic_forecasting_eshybird'

 cache:
  paths:
@@ -32,12 +36,17 @@ before_script:
    </settings>' > $HOME/.m2/settings.xml

 stages:
+  - deployLibrary
  - build
  - deploy

 build:scheduling-abstraction-layer:
  stage: build
  image: $MAVEN_IMAGE_JDK_8
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
  script:
    - $SCHEDULING_ABSTRACTION_LAYER_CLI clean install
  artifacts:
@@ -46,8 +55,12 @@ build:scheduling-abstraction-layer:
      - /builds/melodic/morphemic-preprocessor/maven_repo/org/activeeon/scheduling-abstraction-layer/

 build:amq-message-java-library:
-  stage: build
+  stage: deployLibrary
  image: $MAVEN_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
  script:
    - $AMQ_MESSAGE_JAVA_LIBRARY_CLI clean install
  artifacts:
@@ -55,22 +68,43 @@ build:amq-message-java-library:
    paths:
      - /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/amq-message-java-library/

+build:prediction_orchestrator:
+  stage: build
+  image: $MAVEN_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  script:
+    - $PREDICTON_ORCHESTRATOR_CLI -Pwithout-docker clean install
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - /builds/melodic/morphemic-preprocessor/maven_repo/eu/morphemic/prediction_orchestrator/
+
 build:slo-severity-calculator:
  stage: build
  image: $MAVEN_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
  script:
-    - $SLO_SEVERITY_CALCULATOR_CLI clean install
+    - $SLO_SEVERITY_CALCULATOR_CLI -DfailIfNoTests=false -Dtest=!UnboundedMonitoringAttributeTests,!ConnectivityTests clean install
  artifacts:
    expire_in: 1 week
    paths:
      - /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/SLOSeverityCalculator/

+
+
 deploy:performance-model:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  services:
    - $DOCKER_DIND_SERVICE
  script:
@@ -81,37 +115,39 @@ deploy:performance-model:
    - docker tag performance_model:latest $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH

-deploy:persistent-storage-database:
+deploy:persistent-storage:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  services:
    - $DOCKER_DIND_SERVICE
  script:
    - cd morphemic-persistent-storage/database
-    - docker build -t persistent_storage_database -f ./Dockerfile .
+    - docker build -t persistent_storage -f ./Dockerfile .
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
-    - docker tag persistent_storage_database:latest $CI_REGISTRY_IMAGE/persistent_storage_database:$CI_COMMIT_BRANCH
-    - docker push $CI_REGISTRY_IMAGE/persistent_storage_database:$CI_COMMIT_BRANCH
+    - docker tag persistent_storage:latest $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH

-deploy:persistent-storage-activemq:
+deploy:forecaster-cnn:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  services:
    - $DOCKER_DIND_SERVICE
  script:
-    - cd morphemic-persistent-storage/example
-    - docker build -t persistent_storage_activemq -f ./Dockerfile .
+    - cd forecaster-cnn
+    - docker build -t cnn -f ./Dockerfile .
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
-    - docker tag persistent_storage_activemq:latest $CI_REGISTRY_IMAGE/persistent_storage_activemq:$CI_COMMIT_BRANCH
-    - docker push $CI_REGISTRY_IMAGE/persistent_storage_activemq:$CI_COMMIT_BRANCH
+    - docker tag cnn:latest $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH

 deploy:scheduling-abstraction-layer:
  stage: deploy
@@ -119,6 +155,8 @@ deploy:scheduling-abstraction-layer:
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
+    - proactive-dev
  dependencies:
    - build:scheduling-abstraction-layer
  script:
@@ -130,21 +168,89 @@ deploy:amq-message-java-library:
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  dependencies:
    - build:amq-message-java-library
  script:
    - $AMQ_MESSAGE_JAVA_LIBRARY_CLI deploy

+deploy:prediction_orchestrator:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
+  dependencies:
+    - build:prediction_orchestrator
+  script:
+    - $DOCKER_CLI $PREDICTON_ORCHESTRATOR_CLI deploy
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag $LOCAL_REPO/morphemic/prediction-orchestrator:latest $CI_REGISTRY_IMAGE/prediction-orchestrator:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/prediction-orchestrator:$CI_COMMIT_BRANCH
+
 deploy:slo-severity-calculator:
  stage: deploy
-  image: $MAVEN_IMAGE
+  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  dependencies:
    - build:slo-severity-calculator
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+    - cd morphemic-slo-severity-calculator
+    - mkdir -p target/
+    - cp /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/SLOSeverityCalculator/1.0-SNAPSHOT/SLOSeverityCalculator-1.0-SNAPSHOT.jar target/
+    - docker build -t slo_severity_calculator -f ./Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag slo_severity_calculator:latest $CI_REGISTRY_IMAGE/slo_severity_calculator:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/slo_severity_calculator:$CI_COMMIT_BRANCH
+
+
+
+deploy:exponential_smoothing:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+#    - master
+#    - morphemic-rc1.5
+    - morphemic-rc2.0
+#    - 1synchro-to-2.0-20220127
+
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+#    - apt-get install python3 python3-pip
+    - cd morphemic-forecasting-exponentialsmoothing
+#    - python3 src/setup.py sdist
+    - docker build -t exponential_smoothing -f ./Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag exponential_smoothing:latest $CI_REGISTRY_IMAGE/exponential_smoothing:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/exponential_smoothing:$CI_COMMIT_BRANCH
+
+deploy:ensembler:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
  script:
-    - $SLO_SEVERITY_CALCULATOR_CLI deploy
+    - docker build -t ensembler -f ./deployment/ensembler/Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag ensembler:latest $CI_REGISTRY_IMAGE/ensembler:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/ensembler:$CI_COMMIT_BRANCH
+

 deploy:nbeats:
  stage: deploy
@@ -152,6 +258,7 @@ deploy:nbeats:
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  services:
    - $DOCKER_DIND_SERVICE
  script:
@@ -167,6 +274,7 @@ deploy:tft:
  only:
    - master
    - morphemic-rc1.5
+    - morphemic-rc2.0
  services:
    - $DOCKER_DIND_SERVICE
  script:
@@ -175,3 +283,68 @@ deploy:tft:
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag tft:latest $CI_REGISTRY_IMAGE/tft:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/tft:$CI_COMMIT_BRANCH
+
+deploy:arima:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+    - docker build -t arima -f ./deployment/arima/Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag arima:latest $CI_REGISTRY_IMAGE/arima:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/arima:$CI_COMMIT_BRANCH
+
+deploy:morphemic-forecasting-eshybrid:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+    - cd morphemic-forecasting-eshybrid
+    - docker build -t $DOCKER_FORECAST_ESHYBRID_DOCKER_NAME .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag $DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:latest $CI_REGISTRY_IMAGE/$DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/$DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:$CI_COMMIT_BRANCH
+
+deploy:prophet:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+    - docker build -t prophet -f ./forecasting_prophet/docker_image/Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag prophet:latest $CI_REGISTRY_IMAGE/prophet:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/prophet:$CI_COMMIT_BRANCH
+
+deploy:gluonmachines:
+  stage: deploy
+  image: $DOCKER_DIND_IMAGE
+  only:
+    - master
+    - morphemic-rc1.5
+    - morphemic-rc2.0
+  services:
+    - $DOCKER_DIND_SERVICE
+  script:
+    - docker build -t gluonmachines -f ./forecasting_gluonts/docker_image/Dockerfile .
+    - docker image ls
+    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
+    - docker tag gluonmachines:latest $CI_REGISTRY_IMAGE/gluonmachines:$CI_COMMIT_BRANCH
+    - docker push $CI_REGISTRY_IMAGE/gluonmachines:$CI_COMMIT_BRANCH