.gitlab-ci.yml 9.53 KB
Newer Older
Alicja Reniewicz's avatar
Alicja Reniewicz committed
1
2
3
# Gitlab CI/CD script for the Morphemic Preprocessor project

variables:
4
  MAVEN_IMAGE: "maven:3.8.1-jdk-11"
5
  MAVEN_IMAGE_JDK_8: "maven:3.5.2-jdk-8"
Alicja Reniewicz's avatar
Alicja Reniewicz committed
6
7
  LOCAL_REPO: "127.0.0.1:5000"

8
9
10
11
12
13
  DOCKER_REPO: "gitlab.ow2.org:4567"
  DOCKER_DIND_IMAGE: "docker:19.03.1"
  DOCKER_DIND_SERVICE: "$DOCKER_DIND_IMAGE-dind"
  DOCKER_DRIVER: overlay
  DOCKER_TLS_CERTDIR: "/certs"

mriedl's avatar
mriedl committed
14
  DOCKER_CLI: "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $CI_PROJECT_DIR/maven_repo:/root/.m2 -w /usr/src/mymaven -v $CI_PROJECT_DIR:/usr/src/mymaven $MAVEN_IMAGE"
15
  SCHEDULING_ABSTRACTION_LAYER_CLI: "mvn --batch-mode -f scheduling-abstraction-layer/pom.xml"
16
  AMQ_MESSAGE_JAVA_LIBRARY_CLI: "mvn --batch-mode -f amq-message-java-library/pom.xml"
17
  SLO_SEVERITY_CALCULATOR_CLI: "mvn --batch-mode -f morphemic-slo-severity-calculator/pom.xml"
mriedl's avatar
mriedl committed
18
  PREDICTON_ORCHESTRATOR_CLI: "mvn --batch-mode -N -f prediction_orchestrator/pom.xml"
Alicja Reniewicz's avatar
Alicja Reniewicz committed
19

20
21
  DOCKER_FORECAST_ESHYBRID_DOCKER_NAME: 'morphemic_forecasting_eshybird'

Alicja Reniewicz's avatar
Alicja Reniewicz committed
22
23
24
25
26
27
cache:
  paths:
    - maven_repo/

before_script:
  - echo '=========================================================================='
28
  - echo $MORPHEMIC_PREPROCESSOR_LAYER_CLI
Alicja Reniewicz's avatar
Alicja Reniewicz committed
29
30
31
32
33
34
35
36
37
38
  - echo '=========================================================================='
  - mkdir -p $HOME/.m2
  - echo '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
    https://maven.apache.org/xsd/settings-1.0.0.xsd">
    <localRepository>'$CI_PROJECT_DIR'/maven_repo</localRepository>
    </settings>' > $HOME/.m2/settings.xml

stages:
mriedl's avatar
mriedl committed
39
  - deployLibrary
Alicja Reniewicz's avatar
Alicja Reniewicz committed
40
41
42
  - build
  - deploy

43
build:scheduling-abstraction-layer:
Alicja Reniewicz's avatar
Alicja Reniewicz committed
44
  stage: build
45
  image: $MAVEN_IMAGE_JDK_8
Alicja Reniewicz's avatar
Alicja Reniewicz committed
46
47
48
  script:
    - $SCHEDULING_ABSTRACTION_LAYER_CLI clean install
  artifacts:
49
    expire_in: 1 week
Alicja Reniewicz's avatar
Alicja Reniewicz committed
50
51
52
    paths:
      - /builds/melodic/morphemic-preprocessor/maven_repo/org/activeeon/scheduling-abstraction-layer/

53
build:amq-message-java-library:
mriedl's avatar
mriedl committed
54
  stage: deployLibrary
55
  image: $MAVEN_IMAGE
56
57
58
  only:
    - master
    - morphemic-rc1.5
Ali Fahs's avatar
Ali Fahs committed
59
    - fix-byon-acquire
60
61
62
  script:
    - $AMQ_MESSAGE_JAVA_LIBRARY_CLI clean install
  artifacts:
63
    expire_in: 1 week
64
65
66
    paths:
      - /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/amq-message-java-library/

mriedl's avatar
mriedl committed
67
build:prediction_orchestrator:
68
  stage: build
69
  image: $MAVEN_IMAGE
70
71
72
  only:
    - master
    - morphemic-rc1.5
73
  script:
mriedl's avatar
mriedl committed
74
    - $PREDICTON_ORCHESTRATOR_CLI -Pwithout-docker clean install
75
  artifacts:
76
    expire_in: 1 week
77
    paths:
mriedl's avatar
mriedl committed
78
      - /builds/melodic/morphemic-preprocessor/maven_repo/eu/morphemic/prediction_orchestrator/
79

80
81
build:slo-severity-calculator:
  stage: build
82
  image: $MAVEN_IMAGE
83
84
85
  only:
    - master
    - morphemic-rc1.5
86
  script:
87
    - $SLO_SEVERITY_CALCULATOR_CLI -Dtest=!UnboundedMonitoringAttributeTests,!ConnectivityTests clean install
88
  artifacts:
89
    expire_in: 1 week
90
91
92
    paths:
      - /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/SLOSeverityCalculator/

93
94
95
96
97
98
99
100
101
deploy:performance-model:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
102
103
    - cd morphemic-performance-model/ml_code/
    - docker build -t performance_model -f ./deployment/Dockerfile .
104
105
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
106
    - docker tag performance_model:latest $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH
107
108
    - docker push $CI_REGISTRY_IMAGE/performance_model:$CI_COMMIT_BRANCH

Jean-Didier's avatar
Jean-Didier committed
109
deploy:persistent-storage:
110
111
112
113
114
115
116
117
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
118
    - cd morphemic-persistent-storage/database
Jean-Didier's avatar
Jean-Didier committed
119
    - docker build -t persistent_storage -f ./Dockerfile .
120
121
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
Jean-Didier's avatar
Jean-Didier committed
122
123
    - docker tag persistent_storage:latest $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/persistent_storage:$CI_COMMIT_BRANCH
124

Jean-Didier's avatar
Jean-Didier committed
125
deploy:forecaster-cnn:
126
127
128
129
130
131
132
133
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
Jean-Didier's avatar
Jean-Didier committed
134
135
    - cd forecaster-cnn
    - docker build -t cnn -f ./Dockerfile .
136
137
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
Jean-Didier's avatar
Jean-Didier committed
138
139
    - docker tag cnn:latest $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/cnn:$CI_COMMIT_BRANCH
140

141
deploy:scheduling-abstraction-layer:
Alicja Reniewicz's avatar
Alicja Reniewicz committed
142
  stage: deploy
143
  image: $MAVEN_IMAGE_JDK_8
Alicja Reniewicz's avatar
Alicja Reniewicz committed
144
145
146
  only:
    - master
    - morphemic-rc1.5
147
    - proactive-dev
Ali Fahs's avatar
Ali Fahs committed
148
    - fix-byon-acquire
149
150
  dependencies:
    - build:scheduling-abstraction-layer
Alicja Reniewicz's avatar
Alicja Reniewicz committed
151
  script:
152
    - $SCHEDULING_ABSTRACTION_LAYER_CLI deploy
153
154
155
156
157
158
159
160
161
162
163

deploy:amq-message-java-library:
  stage: deploy
  image: $MAVEN_IMAGE
  only:
    - master
    - morphemic-rc1.5
  dependencies:
    - build:amq-message-java-library
  script:
    - $AMQ_MESSAGE_JAVA_LIBRARY_CLI deploy
164

165
deploy:prediction_orchestrator:
Fotis Paraskevopoulos's avatar
Fotis Paraskevopoulos committed
166
167
168
169
170
171
172
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
173
174
175
  dependencies:
    - build:prediction_orchestrator
  script:
mriedl's avatar
mriedl committed
176
    - $DOCKER_CLI $PREDICTON_ORCHESTRATOR_CLI deploy
mriedl's avatar
mriedl committed
177
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
mriedl's avatar
mriedl committed
178
179
    - docker tag $LOCAL_REPO/morphemic/prediction-orchestrator:latest $CI_REGISTRY_IMAGE/prediction-orchestrator:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/prediction-orchestrator:$CI_COMMIT_BRANCH
180

181
182
deploy:slo-severity-calculator:
  stage: deploy
183
  image: $DOCKER_DIND_IMAGE
184
185
186
187
188
  only:
    - master
    - morphemic-rc1.5
  dependencies:
    - build:slo-severity-calculator
189
190
  services:
    - $DOCKER_DIND_SERVICE
191
  script:
192
    - cd morphemic-slo-severity-calculator
193
194
    - mkdir -p target/
    - cp /builds/melodic/morphemic-preprocessor/maven_repo/gr/ntua/imu/morphemic/SLOSeverityCalculator/1.0-SNAPSHOT/SLOSeverityCalculator-1.0-SNAPSHOT.jar target/
195
196
197
198
199
    - docker build -t slo_severity_calculator -f ./Dockerfile .
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag slo_severity_calculator:latest $CI_REGISTRY_IMAGE/slo_severity_calculator:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/slo_severity_calculator:$CI_COMMIT_BRANCH
200
201
202
203
204
205
206
207
208
209

deploy:nbeats:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
210
    - docker build -t nbeats -f ./deployment/nbeats/Dockerfile .
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag nbeats:latest $CI_REGISTRY_IMAGE/nbeats:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/nbeats:$CI_COMMIT_BRANCH

deploy:tft:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
225
    - docker build -t tft -f ./deployment/tft/Dockerfile .
226
227
228
229
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag tft:latest $CI_REGISTRY_IMAGE/tft:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/tft:$CI_COMMIT_BRANCH
Fotis Paraskevopoulos's avatar
Fotis Paraskevopoulos committed
230

Anna Warno's avatar
Anna Warno committed
231
232
233
234
235
236
237
238
239
240
241
242
243
244
deploy:arima:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
    - docker build -t arima -f ./deployment/arima/Dockerfile .
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag arima:latest $CI_REGISTRY_IMAGE/arima:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/arima:$CI_COMMIT_BRANCH
245

246
deploy:morphemic-forecasting-eshybrid:
247
248
249
250
251
252
253
254
255
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
    - cd morphemic-forecasting-eshybrid
256
    - docker build -t $DOCKER_FORECAST_ESHYBRID_DOCKER_NAME .
257
258
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
259
260
    - docker tag $DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:latest $CI_REGISTRY_IMAGE/$DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/$DOCKER_FORECAST_ESHYBRID_DOCKER_NAME:$CI_COMMIT_BRANCH
261
262
263
264
265
266
267
268
269
270

deploy:prophet:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
Diana Jlailaty's avatar
Diana Jlailaty committed
271
    - docker build -t prophet -f ./forecasting_prophet/docker_image/Dockerfile .
272
273
274
275
276
277
278
279
280
281
282
283
284
285
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag prophet:latest $CI_REGISTRY_IMAGE/prophet:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/prophet:$CI_COMMIT_BRANCH
    
deploy:gluonmachines:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - master
    - morphemic-rc1.5
  services:
    - $DOCKER_DIND_SERVICE
  script:
Diana Jlailaty's avatar
Diana Jlailaty committed
286
    - docker build -t gluonmachines -f ./forecasting_gluonts/docker_image/Dockerfile .
287
288
289
290
    - docker image ls
    - echo "$K8S_SECRET_DOCKER_PASSWORD" | docker login $CI_REGISTRY -u $K8S_SECRET_DOCKER_USER --password-stdin
    - docker tag gluonmachines:latest $CI_REGISTRY_IMAGE/gluonmachines:$CI_COMMIT_BRANCH
    - docker push $CI_REGISTRY_IMAGE/gluonmachines:$CI_COMMIT_BRANCH