.gitlab-ci.yml 3.62 KB
Newer Older
1
2
3
# ZPP Ensembler GitLab CI config

variables:
Sebastian Miller's avatar
Sebastian Miller committed
4
5
  SYSTEM_SETUP_DIR: $CI_PROJECT_DIR/integration/system-setup
  ENSEMBLER_DIR: $CI_PROJECT_DIR/integration/ensembler_flask
6
  DOCKER_DIND_IMAGE: "docker:20.10.14"
7
8
  DOCKER_DIND_SERVICE: "$DOCKER_DIND_IMAGE-dind"
  CI_REGISTY: "gitlab.ow2.org:4567"
9
10
  CONTAINER_BASE_TAG: $CI_REGISTRY_IMAGE/ensembler
  CONTAINER_RELEASE_IMAGE: $CONTAINER_BASE_TAG:$CI_COMMIT_BRANCH
11
12

stages:
13
  - unit-tests
14
  - build
15
  - integration-tests
16
17
  - deploy

18
19
20
21
22
23
24
# Perform unit tests
unit-tests:ensembler:
  stage: unit-tests
  image: python:3.8-slim-buster
  only:
    - integration
  script:
25
    - cd $ENSEMBLER_DIR
26
27
28
29
30
31
32
    - pip install --no-cache-dir --upgrade pip
    - pip install --no-cache-dir poetry
    - poetry install
    - UNIT_TESTS_MODE='True' poetry run pytest tests/test_predictor.py
    - UNIT_TESTS_MODE='True' poetry run pytest tests/test_model_persistence.py

# Build Docker image
33
34
35
36
37
38
39
40
build:ensembler:      
  stage: build
  image: $DOCKER_DIND_IMAGE
  only:
    - integration
  services:
    - $DOCKER_DIND_SERVICE
  script:
41
    - cd $ENSEMBLER_DIR
42
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
43
    - docker pull $CONTAINER_RELEASE_IMAGE || true   # Attempt to pull previous image as a cache for the below build.
44
    - docker build --cache-from $CONTAINER_RELEASE_IMAGE -t ensembler .
45
    - docker image ls
46
    - cd $CI_PROJECT_DIR
Sebastian Miller's avatar
Sebastian Miller committed
47
    - mkdir image
Sebastian Miller's avatar
Sebastian Miller committed
48
    - docker save ensembler > image/ensembler.tar  # Save the image as an artifact for further stages
49
50
51
  artifacts:
    paths:
      - image
52

53
# Test requests sent to endembler
54
55
integration-tests:ensembler:
  stage: integration-tests
56
  image: $DOCKER_DIND_IMAGE
57
58
59
  only:
    - integration
  services:
60
    - $DOCKER_DIND_SERVICE
61
62
63
64
65
66
67
    - name: rmohr/activemq:5.15.9
      alias: activemq
    - name: influxdb:1.8.4
      alias: influxdb
      variables:
        INFLUXDB_ADMIN_USER: admin
        INFLUXDB_ADMIN_PASSWORD: admin1234
68
  script:
69
70
71
    # Install essential packages
    - apk add --update --no-cache curl python3 py3-pip alpine-sdk
    # Create appripriate databases in InfluxDB
72
73
    - 'curl --user admin:admin1234 -XPOST "http://influxdb:8086/query" --data-urlencode "q=CREATE DATABASE melodic_ui"'
    - 'curl --user admin:admin1234 -XPOST "http://influxdb:8086/query" --data-urlencode "q=CREATE DATABASE morphemic"'
Sebastian Miller's avatar
Sebastian Miller committed
74
75
76
    # Load the artifact image
    - docker load -i image/ensembler.tar
    # Start the ensembler
Sebastian Miller's avatar
Sebastian Miller committed
77
    - docker run 
78
        -d 
Sebastian Miller's avatar
Sebastian Miller committed
79
        --env-file $ENSEMBLER_DIR/tests/config/server_params.env 
80
81
        --add-host=activemq:$(getent hosts activemq  | awk '{ print $1 }') 
        --add-host=influxdb:$(getent hosts influxdb  | awk '{ print $1 }') 
Sebastian Miller's avatar
Sebastian Miller committed
82
        -v $ENSEMBLER_DIR/tests/config/properties.yml:/ensembler/config/properties.yml 
83
        --name ensembler 
Sebastian Miller's avatar
Sebastian Miller committed
84
        ensembler
85
    - cd $SYSTEM_SETUP_DIR/influxdb
Sebastian Miller's avatar
Sebastian Miller committed
86
87
    - chmod +x ./fill_db.sh
    # Upload test data to InfluxDB
88
    - ./fill_db.sh 'influxdb:8086' 'sample_data_separate_runs'
Sebastian Miller's avatar
Sebastian Miller committed
89
    - cd $ENSEMBLER_DIR
90
    # Install essential python packages
Sebastian Miller's avatar
Sebastian Miller committed
91
    - pip3 install --no-cache-dir --upgrade pip setuptools wheel
92
    - pip3 install --no-cache-dir poetry
93
    - poetry install
Sebastian Miller's avatar
Sebastian Miller committed
94
95
    # Run the requests test
    - ENSEMBLER_HOST_IP='127.0.0.1' poetry run pytest tests/test_requests.py
96
    - docker stop ensembler
97

98
# Push the built and tested image
99
100
101
102
103
104
105
106
deploy:ensembler:
  stage: deploy
  image: $DOCKER_DIND_IMAGE
  only:
    - integration
  services:
    - $DOCKER_DIND_SERVICE
  script:
107
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
108
109
    - docker load -i image/ensembler.tar   # Load the artifact image
    - docker tag ensembler $CONTAINER_RELEASE_IMAGE
110
    - docker push $CONTAINER_RELEASE_IMAGE