unique containerization of input api and influxdb

morphemic-persistent-storage/influxdb/env.influxdb → morphemic-persistent-storage/database/.env

+# InfluxDB setup details.
 INFLUXDB_DATA_ENGINE=tsm1
 INFLUXDB_REPORTING_DISABLED=false
 INFLUXDB_ADMIN_USER=admin 
@@ -6,3 +7,6 @@ INFLUXDB_DB=morphemic
 INFLUXDB_USER=morphemic
 INFLUXDB_USER_PASSWORD=password
 INFLUXDB_HTTP_FLUX_ENABLED=true
+
+# Python API message queue connection.
+ACTIVEMQ_HOSTNAME=activemq
\ No newline at end of file

morphemic-persistent-storage/database/Dockerfile

+# --- Stage 1: API Dependency Loader
+
+# :: Initial dependency loading image.
+FROM python:3.7-slim as api-loader
+
+# Get package dependencies.
+COPY inputapi/requirements.txt .
+RUN pip install --user -r requirements.txt
+
+# --- Stage 2: Combined InfluxDB + Python API Image
+FROM python:3.7-alpine3.12
+
+# ::  InfluxDB standard setup
+RUN echo 'hosts: files dns' >> /etc/nsswitch.conf
+RUN apk add --no-cache tzdata bash ca-certificates && \
+    update-ca-certificates
+
+ENV INFLUXDB_VERSION 1.8.4
+RUN set -ex && \
+    mkdir ~/.gnupg; \
+    echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf; \
+    apk add --no-cache --virtual .build-deps wget gnupg tar && \
+    for key in \
+        05CE15085FC09D18E99EFB22684A14CF2582E0C5 ; \
+    do \
+        gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
+        gpg --keyserver pgp.mit.edu --recv-keys "$key" || \
+        gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \
+    done && \
+    wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc && \
+    wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
+    gpg --batch --verify influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
+    mkdir -p /usr/src && \
+    tar -C /usr/src -xzf influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
+    rm -f /usr/src/influxdb-*/influxdb.conf && \
+    chmod +x /usr/src/influxdb-*/* && \
+    cp -a /usr/src/influxdb-*/* /usr/bin/ && \
+    gpgconf --kill all && \
+    rm -rf *.tar.gz* /usr/src /root/.gnupg && \
+    apk del .build-deps
+COPY influxdb.conf /etc/influxdb/influxdb.conf
+
+EXPOSE 8086
+
+VOLUME /var/lib/influxdb
+
+COPY entrypoint.sh /entrypoint.sh
+COPY init-influxdb.sh /init-influxdb.sh
+
+
+# :: Python API setup
+# Copy compiled dependencies from
+# the standard user pip directory 
+# and update PATH.
+COPY --from=api-loader /root/.local /root/.local
+ENV PATH=/root/.local:$PATH
+
+# Copy Python API.
+RUN mkdir inputapi
+COPY ./inputapi/src ./inputapi/
+
+# Execute both in entrypoint.sh.
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["influxd"]

morphemic-persistent-storage/database/README.md

+# Time Series Database
+
+This component extends the standard InfluxDB Alpine Dockerfile with Python support and the local Input API.
+
+## Maintenance
+
+Check comments on the main Dockerfile.

morphemic-persistent-storage/database/entrypoint.sh

+#!/bin/bash
+set -e
+
+if [ "${1:0:1}" = '-' ]; then
+    set -- influxd "$@"
+fi
+
+if [ "$1" = 'influxd' ]; then
+	/init-influxdb.sh "${@:2}"
+fi
+
+# Run API
+python inputapi/app.py &
+
+# Run InfluxDB
+exec "$@"

morphemic-persistent-storage/database/influxdb.conf

+[meta]
+  dir = "/var/lib/influxdb/meta"
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  engine = "tsm1"
+  wal-dir = "/var/lib/influxdb/wal"

morphemic-persistent-storage/database/init-influxdb.sh

+#!/bin/bash
+set -e
+
+AUTH_ENABLED="$INFLUXDB_HTTP_AUTH_ENABLED"
+
+if [ -z "$AUTH_ENABLED" ]; then
+	AUTH_ENABLED="$(grep -iE '^\s*auth-enabled\s*=\s*true' /etc/influxdb/influxdb.conf | grep -io 'true' | cat)"
+else
+	AUTH_ENABLED="$(echo "$INFLUXDB_HTTP_AUTH_ENABLED" | grep -io 'true' | cat)"
+fi
+
+INIT_USERS=$([ ! -z "$AUTH_ENABLED" ] && [ ! -z "$INFLUXDB_ADMIN_USER" ] && echo 1 || echo)
+
+# Check if an environment variable for where to put meta is set.
+# If so, then use that directory, otherwise use the default.
+if [ -z "$INFLUXDB_META_DIR" ]; then
+	META_DIR="/var/lib/influxdb/meta"
+else
+	META_DIR="$INFLUXDB_META_DIR"
+fi
+
+if ( [ ! -z "$INIT_USERS" ] || [ ! -z "$INFLUXDB_DB" ] || [ "$(ls -A /docker-entrypoint-initdb.d 2> /dev/null)" ] ) && [ ! "$(ls -d "$META_DIR" 2>/dev/null)" ]; then
+
+	INIT_QUERY=""
+	CREATE_DB_QUERY="CREATE DATABASE $INFLUXDB_DB"
+
+	if [ ! -z "$INIT_USERS" ]; then
+
+		if [ -z "$INFLUXDB_ADMIN_PASSWORD" ]; then
+			INFLUXDB_ADMIN_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
+			echo "INFLUXDB_ADMIN_PASSWORD:$INFLUXDB_ADMIN_PASSWORD"
+		fi
+
+		INIT_QUERY="CREATE USER \"$INFLUXDB_ADMIN_USER\" WITH PASSWORD '$INFLUXDB_ADMIN_PASSWORD' WITH ALL PRIVILEGES"
+	elif [ ! -z "$INFLUXDB_DB" ]; then
+		INIT_QUERY="$CREATE_DB_QUERY"
+	else
+		INIT_QUERY="SHOW DATABASES"
+	fi
+
+	INFLUXDB_INIT_PORT="8086"
+
+	INFLUXDB_HTTP_BIND_ADDRESS=127.0.0.1:$INFLUXDB_INIT_PORT INFLUXDB_HTTP_HTTPS_ENABLED=false influxd "$@" &
+	pid="$!"
+
+	INFLUX_CMD="influx -host 127.0.0.1 -port $INFLUXDB_INIT_PORT -execute "
+
+	for i in {30..0}; do
+		if $INFLUX_CMD "$INIT_QUERY" &> /dev/null; then
+			break
+		fi
+		echo 'influxdb init process in progress...'
+		sleep 1
+	done
+
+	if [ "$i" = 0 ]; then
+		echo >&2 'influxdb init process failed.'
+		exit 1
+	fi
+
+	if [ ! -z "$INIT_USERS" ]; then
+
+		INFLUX_CMD="influx -host 127.0.0.1 -port $INFLUXDB_INIT_PORT -username ${INFLUXDB_ADMIN_USER} -password ${INFLUXDB_ADMIN_PASSWORD} -execute "
+
+		if [ ! -z "$INFLUXDB_DB" ]; then
+			$INFLUX_CMD "$CREATE_DB_QUERY"
+		fi
+
+		if [ ! -z "$INFLUXDB_USER" ] && [ -z "$INFLUXDB_USER_PASSWORD" ]; then
+			INFLUXDB_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
+			echo "INFLUXDB_USER_PASSWORD:$INFLUXDB_USER_PASSWORD"
+		fi
+
+		if [ ! -z "$INFLUXDB_USER" ]; then
+			$INFLUX_CMD "CREATE USER \"$INFLUXDB_USER\" WITH PASSWORD '$INFLUXDB_USER_PASSWORD'"
+
+			$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_USER\""
+
+			if [ ! -z "$INFLUXDB_DB" ]; then
+				$INFLUX_CMD "GRANT ALL ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_USER\""
+			fi
+		fi
+
+		if [ ! -z "$INFLUXDB_WRITE_USER" ] && [ -z "$INFLUXDB_WRITE_USER_PASSWORD" ]; then
+			INFLUXDB_WRITE_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
+			echo "INFLUXDB_WRITE_USER_PASSWORD:$INFLUXDB_WRITE_USER_PASSWORD"
+		fi
+
+		if [ ! -z "$INFLUXDB_WRITE_USER" ]; then
+			$INFLUX_CMD "CREATE USER \"$INFLUXDB_WRITE_USER\" WITH PASSWORD '$INFLUXDB_WRITE_USER_PASSWORD'"
+			$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_WRITE_USER\""
+
+			if [ ! -z "$INFLUXDB_DB" ]; then
+				$INFLUX_CMD "GRANT WRITE ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_WRITE_USER\""
+			fi
+		fi
+
+		if [ ! -z "$INFLUXDB_READ_USER" ] && [ -z "$INFLUXDB_READ_USER_PASSWORD" ]; then
+			INFLUXDB_READ_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
+			echo "INFLUXDB_READ_USER_PASSWORD:$INFLUXDB_READ_USER_PASSWORD"
+		fi
+
+		if [ ! -z "$INFLUXDB_READ_USER" ]; then
+			$INFLUX_CMD "CREATE USER \"$INFLUXDB_READ_USER\" WITH PASSWORD '$INFLUXDB_READ_USER_PASSWORD'"
+			$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_READ_USER\""
+
+			if [ ! -z "$INFLUXDB_DB" ]; then
+				$INFLUX_CMD "GRANT READ ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_READ_USER\""
+			fi
+		fi
+
+	fi
+
+	for f in /docker-entrypoint-initdb.d/*; do
+		case "$f" in
+			*.sh)     echo "$0: running $f"; . "$f" ;;
+			*.iql)    echo "$0: running $f"; $INFLUX_CMD "$(cat ""$f"")"; echo ;;
+			*)        echo "$0: ignoring $f" ;;
+		esac
+		echo
+	done
+
+	if ! kill -s TERM "$pid" || ! wait "$pid"; then
+		echo >&2 'influxdb init process failed. (Could not stop influxdb)'
+		exit 1
+	fi
+
+fi

morphemic-persistent-storage/database/inputapi/README.md

+# Input API
+The component which handles message queue inputs to InfluxDB within the Persistent Storage Module of the [MORPHEMIC](https://www.morphemic.cloud) platform.
\ No newline at end of file

morphemic-persistent-storage/database/inputapi/requirements.txt

+flask
+influxdb
+stomp.py
+requests
\ No newline at end of file

morphemic-persistent-storage/database/inputapi/setup.cfg

+[metadata]
+name = morphemic-inputapi
+version = 1.0.0
+author = Jean-Didier Totow
+author_email = totow@unipi.gr
+description = The InputAPI component for the Persistent Storage module of the MORPHEMIC platform.
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://git.dac.ds.unipi.gr/morphemic/persistent-storage
+project_urls =
+    Bug Tracker = https://git.dac.ds.unipi.gr/morphemic/persistent-storage/-/issues
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+python_requires = >=3.7
+
+[options.packages.find]
+where = src
\ No newline at end of file

morphemic-persistent-storage/inputapi/src/activemqlistermanager.py → morphemic-persistent-storage/database/inputapi/src/activemqlistermanager.py

 import stomp, os, json, time
 from threading import Thread
 
-data_format = os.environ.get("DATA_FORMAT","json")
+data_format = os.environ.get("DATA_FORMAT", "json")
+
 
 class Listener(object):
-    def __init__(self, conn,handler):
+    def __init__(self, conn, handler):
         self.conn = conn
         self.count = 0
         self.handler = handler
         self.start = time.time()
 
     def on_error(self, headers, message):
-        print('received an error %s' % message)
+        print("received an error %s" % message)
 
     def on_message(self, headers, message):
         self.handler(message)
 
+
 class Worker(Thread):
-    def __init__(self,hostname,port, username, password, topic, handler, sleeping, index):
+    def __init__(
+        self, hostname, port, username, password, topic, handler, sleeping, index
+    ):
         self.hostname = hostname
         self.port = port
         self.topic = topic
@@ -41,16 +45,20 @@ class Worker(Thread):
 
     def run(self):
         print("Worker {0} started".format(self.index))
-        print("Hostname : {0}\nPort: {1}\nTopic: {2}".format(self.hostname,self.port,self.topic))
+        print(
+            "Hostname : {0}\nPort: {1}\nTopic: {2}".format(
+                self.hostname, self.port, self.topic
+            )
+        )
         while True:
             if self.normal_stop:
                 break
             print("Trying to connect ...")
             try:
-                conn = stomp.Connection(host_and_ports = [(self.hostname, self.port)])
-                conn.set_listener('', Listener(conn,self.handler))
-                conn.connect(login=self.username,passcode=self.password)
-                conn.subscribe(destination=self.topic, id=1, ack='auto')
+                conn = stomp.Connection(host_and_ports=[(self.hostname, self.port)])
+                conn.set_listener("", Listener(conn, self.handler))
+                conn.connect(login=self.username, passcode=self.password)
+                conn.subscribe(destination=self.topic, id=1, ack="auto")
                 self.status = "started"
                 print("Waiting for messages...")
                 while 1:
@@ -66,13 +74,14 @@ class Worker(Thread):
         self.status = "stopped"
 
 
-class ActiveMQManager():
+class ActiveMQManager:
     def __init__(self, handler):
         self.all_threads = []
         self.handler = handler
         thread_controller = Thread(target=self.workerController)
         thread_controller.start()
-    def getData(self,data):
+
+    def getData(self, data):
         if data_format == "json":
             _data = None
             try:
@@ -94,14 +103,14 @@ class ActiveMQManager():
                     w.start()
             time.sleep(20)
 
-    def startWorker(self,hostname,port, username, password, topic, key):
+    def startWorker(self, hostname, port, username, password, topic, key):
         for w in self.all_threads:
             if w.getIndex() == key:
                 print("Connection already registered")
                 return None
-        sleeping = 5 #5 seconds 
-        worker = Worker(hostname,port,username,password,topic,self.handler,sleeping,key) 
+        sleeping = 5  # 5 seconds
+        worker = Worker(
+            hostname, port, username, password, topic, self.handler, sleeping, key
+        )
         self.all_threads.append(worker)
         worker.start()
-
-        
\ No newline at end of file

morphemic-persistent-storage/inputapi/src/listener.py → morphemic-persistent-storage/database/inputapi/src/listener.py

@@ -27,15 +27,15 @@ host = os.getenv("ACTIVEMQ_HOST") or "localhost"
 port = os.getenv("ACTIVEMQ_PORT") or 61610
 destination = "/topic/ZZZ"
 
-class MyListener(object):
 
+class MyListener(object):
     def __init__(self, conn):
         self.conn = conn
         self.count = 0
         self.start = time.time()
 
     def on_error(self, headers, message):
-        print('received an error %s' % message)
+        print("received an error %s" % message)
 
     def on_message(self, headers, message):
         print(message)
@@ -55,10 +55,11 @@ class MyListener(object):
             print("Received %s messages." % self.count)
         """
 
-conn = stomp.Connection(host_and_ports = [(host, port)])
-conn.set_listener('', MyListener(conn))
-conn.connect(login=user,passcode=password)
-conn.subscribe(destination=destination, id=1, ack='auto')
+
+conn = stomp.Connection(host_and_ports=[(host, port)])
+conn.set_listener("", MyListener(conn))
+conn.connect(login=user, passcode=password)
+conn.subscribe(destination=destination, id=1, ack="auto")
 print("Waiting for messages...")
 while 1:
     time.sleep(10)

morphemic-persistent-storage/docker-compose.yaml

 version: '2'
 
 services:
-  influxdb:
-    image: influxdb:1.8
-    container_name: influxdb
-    # COMMENTING OUT PORTS FOR NETWORK ISOLATION
-    ports:
-    - "8083:8083"
-    - "8086:8086"
-    - "8090:8090"
+  database:
+    build:
+      context: ./database
+    container_name: database
+    restart: always
     env_file:
-      - './influxdb/env.influxdb'
+      - "./database/.env"
     volumes:
-      - './influxdb/data:/var/lib/influxdb'
-
-  performance_model:
-    image: jdtotow/performance_model
-    container_name: performance_model
-    ports:
-    - 8766:8766
-    - 8767:8767
-    volumes: 
-      - './performance_model/datasets:/var/performance_model/datasets:Z'
-      - './performance_model/train:/var/performance_model/train:Z'
-      - './performance_model/logs:/var/performance_model/logs:Z'
-      - './performance_model/models:/var/performance_model/models:Z'
-      #- './performance_model/db:/var/performance_model/db:Z'
-      #- './performance_model/uwsgi.ini:/app/uwsgi.ini'
-    environment: 
-      - "PERFORMANCE_MODEL_URL=http://localhost:8766/api/v1/train"
-      - "INFLUXDB_HOSTNAME=influxdb"
-      - "ACTIVEMQ_HOST=activemq"
-      - "DATASET_LOCAL_REPOSITORY=/var/performance_model/datasets"
-      - "MLMODELPATH=/var/performance_model/models"
-      - "TRAIN_DATA_FOLDER=/var/performance_model/train"
-      - "LOGS_FOLDER=/var/performance_model/logs"
-      #- "LOCAL_DATABASE_PATH=/var/performance_model/db/"
-  
-  inputapi:
-    image: jdtotow/inputapi 
-    container_name: inputapi
-    restart: always 
-    environment: 
-    - "INFLUXDB_URL=http://influxdb:8086"
-    - "INFLUXDB_HOSTNAME=influxdb"
-    - "ACTIVEMQ_HOSTNAME=activemq"
-
+      - "./database/data:/var/lib/influxdb"
   publisher:
     image: jdtotow/publisher
     container_name: publisher
@@ -96,5 +60,3 @@ services:
   #  container_name: prometheus
   #  volumes:
   #  - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-
-

morphemic-persistent-storage/env.grafana

-GF_INSTALL_PLUGINS=grafana-clock-panel,briangann-gauge-panel,natel-plotly-panel,grafana-simple-json-datasource
\ No newline at end of file

morphemic-persistent-storage/inputapi/Dockerfile

-FROM python:3.7 
-RUN pip install --upgrade pip 
-RUN pip install flask influxdb stomp.py requests 
-RUN mkdir /app 
-COPY src/ /app
-RUN useradd per_storage 
-USER per_storage
-WORKDIR /app
-CMD ["python","-u","app.py"]
\ No newline at end of file

morphemic-persistent-storage/inputapi/build-inputapi.sh

-#!/bin/bash
-
-sudo docker build -t jdtotow/inputapi .
-sudo docker push jdtotow/inputapi  
\ No newline at end of file

morphemic-persistent-storage/outputapi/Dockerfile

-# This is the Dockerfile for the OutputAPI of the Persistent Storage Module.
-# -----
-# Check the README for more information regarding this API.
-
-# Build the Go binary.
-FROM golang:alpine AS build
-WORKDIR /app
-ADD ./src /app
-RUN go mod download && go build -o main .
-
-# Migrate the binary to the Alpine image and execute.
-FROM alpine
-RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
-WORKDIR /app
-COPY --from=build /app/main /app
-EXPOSE 7764
-ENTRYPOINT ./main
\ No newline at end of file