Commit 45d73b04 authored by Jean-Didier Totow's avatar Jean-Didier Totow
Browse files

unique containerization of input api and influxdb

parent 8e455ba8
# InfluxDB setup details.
INFLUXDB_DATA_ENGINE=tsm1 INFLUXDB_DATA_ENGINE=tsm1
INFLUXDB_REPORTING_DISABLED=false INFLUXDB_REPORTING_DISABLED=false
INFLUXDB_ADMIN_USER=admin INFLUXDB_ADMIN_USER=admin
...@@ -5,4 +6,7 @@ INFLUXDB_ADMIN_PASSWORD=password ...@@ -5,4 +6,7 @@ INFLUXDB_ADMIN_PASSWORD=password
INFLUXDB_DB=morphemic INFLUXDB_DB=morphemic
INFLUXDB_USER=morphemic INFLUXDB_USER=morphemic
INFLUXDB_USER_PASSWORD=password INFLUXDB_USER_PASSWORD=password
INFLUXDB_HTTP_FLUX_ENABLED=true INFLUXDB_HTTP_FLUX_ENABLED=true
\ No newline at end of file
# Python API message queue connection.
ACTIVEMQ_HOSTNAME=activemq
\ No newline at end of file
# --- Stage 1: API Dependency Loader
# :: Initial dependency loading image.
FROM python:3.7-slim as api-loader
# Get package dependencies.
COPY inputapi/requirements.txt .
RUN pip install --user -r requirements.txt
# --- Stage 2: Combined InfluxDB + Python API Image
FROM python:3.7-alpine3.12
# :: InfluxDB standard setup
RUN echo 'hosts: files dns' >> /etc/nsswitch.conf
RUN apk add --no-cache tzdata bash ca-certificates && \
update-ca-certificates
ENV INFLUXDB_VERSION 1.8.4
RUN set -ex && \
mkdir ~/.gnupg; \
echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf; \
apk add --no-cache --virtual .build-deps wget gnupg tar && \
for key in \
05CE15085FC09D18E99EFB22684A14CF2582E0C5 ; \
do \
gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
gpg --keyserver pgp.mit.edu --recv-keys "$key" || \
gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \
done && \
wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc && \
wget --no-verbose https://dl.influxdata.com/influxdb/releases/influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
gpg --batch --verify influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz.asc influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
mkdir -p /usr/src && \
tar -C /usr/src -xzf influxdb-${INFLUXDB_VERSION}-static_linux_amd64.tar.gz && \
rm -f /usr/src/influxdb-*/influxdb.conf && \
chmod +x /usr/src/influxdb-*/* && \
cp -a /usr/src/influxdb-*/* /usr/bin/ && \
gpgconf --kill all && \
rm -rf *.tar.gz* /usr/src /root/.gnupg && \
apk del .build-deps
COPY influxdb.conf /etc/influxdb/influxdb.conf
EXPOSE 8086
VOLUME /var/lib/influxdb
COPY entrypoint.sh /entrypoint.sh
COPY init-influxdb.sh /init-influxdb.sh
# :: Python API setup
# Copy compiled dependencies from
# the standard user pip directory
# and update PATH.
COPY --from=api-loader /root/.local /root/.local
ENV PATH=/root/.local:$PATH
# Copy Python API.
RUN mkdir inputapi
COPY ./inputapi/src ./inputapi/
# Execute both in entrypoint.sh.
ENTRYPOINT ["/entrypoint.sh"]
CMD ["influxd"]
# Time Series Database
This component extends the standard InfluxDB Alpine Dockerfile with Python support and the local Input API.
## Maintenance
Check comments on the main Dockerfile.
#!/bin/bash
set -e
if [ "${1:0:1}" = '-' ]; then
set -- influxd "$@"
fi
if [ "$1" = 'influxd' ]; then
/init-influxdb.sh "${@:2}"
fi
# Run API
python inputapi/app.py &
# Run InfluxDB
exec "$@"
[meta]
dir = "/var/lib/influxdb/meta"
[data]
dir = "/var/lib/influxdb/data"
engine = "tsm1"
wal-dir = "/var/lib/influxdb/wal"
#!/bin/bash
set -e
AUTH_ENABLED="$INFLUXDB_HTTP_AUTH_ENABLED"
if [ -z "$AUTH_ENABLED" ]; then
AUTH_ENABLED="$(grep -iE '^\s*auth-enabled\s*=\s*true' /etc/influxdb/influxdb.conf | grep -io 'true' | cat)"
else
AUTH_ENABLED="$(echo "$INFLUXDB_HTTP_AUTH_ENABLED" | grep -io 'true' | cat)"
fi
INIT_USERS=$([ ! -z "$AUTH_ENABLED" ] && [ ! -z "$INFLUXDB_ADMIN_USER" ] && echo 1 || echo)
# Check if an environment variable for where to put meta is set.
# If so, then use that directory, otherwise use the default.
if [ -z "$INFLUXDB_META_DIR" ]; then
META_DIR="/var/lib/influxdb/meta"
else
META_DIR="$INFLUXDB_META_DIR"
fi
if ( [ ! -z "$INIT_USERS" ] || [ ! -z "$INFLUXDB_DB" ] || [ "$(ls -A /docker-entrypoint-initdb.d 2> /dev/null)" ] ) && [ ! "$(ls -d "$META_DIR" 2>/dev/null)" ]; then
INIT_QUERY=""
CREATE_DB_QUERY="CREATE DATABASE $INFLUXDB_DB"
if [ ! -z "$INIT_USERS" ]; then
if [ -z "$INFLUXDB_ADMIN_PASSWORD" ]; then
INFLUXDB_ADMIN_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
echo "INFLUXDB_ADMIN_PASSWORD:$INFLUXDB_ADMIN_PASSWORD"
fi
INIT_QUERY="CREATE USER \"$INFLUXDB_ADMIN_USER\" WITH PASSWORD '$INFLUXDB_ADMIN_PASSWORD' WITH ALL PRIVILEGES"
elif [ ! -z "$INFLUXDB_DB" ]; then
INIT_QUERY="$CREATE_DB_QUERY"
else
INIT_QUERY="SHOW DATABASES"
fi
INFLUXDB_INIT_PORT="8086"
INFLUXDB_HTTP_BIND_ADDRESS=127.0.0.1:$INFLUXDB_INIT_PORT INFLUXDB_HTTP_HTTPS_ENABLED=false influxd "$@" &
pid="$!"
INFLUX_CMD="influx -host 127.0.0.1 -port $INFLUXDB_INIT_PORT -execute "
for i in {30..0}; do
if $INFLUX_CMD "$INIT_QUERY" &> /dev/null; then
break
fi
echo 'influxdb init process in progress...'
sleep 1
done
if [ "$i" = 0 ]; then
echo >&2 'influxdb init process failed.'
exit 1
fi
if [ ! -z "$INIT_USERS" ]; then
INFLUX_CMD="influx -host 127.0.0.1 -port $INFLUXDB_INIT_PORT -username ${INFLUXDB_ADMIN_USER} -password ${INFLUXDB_ADMIN_PASSWORD} -execute "
if [ ! -z "$INFLUXDB_DB" ]; then
$INFLUX_CMD "$CREATE_DB_QUERY"
fi
if [ ! -z "$INFLUXDB_USER" ] && [ -z "$INFLUXDB_USER_PASSWORD" ]; then
INFLUXDB_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
echo "INFLUXDB_USER_PASSWORD:$INFLUXDB_USER_PASSWORD"
fi
if [ ! -z "$INFLUXDB_USER" ]; then
$INFLUX_CMD "CREATE USER \"$INFLUXDB_USER\" WITH PASSWORD '$INFLUXDB_USER_PASSWORD'"
$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_USER\""
if [ ! -z "$INFLUXDB_DB" ]; then
$INFLUX_CMD "GRANT ALL ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_USER\""
fi
fi
if [ ! -z "$INFLUXDB_WRITE_USER" ] && [ -z "$INFLUXDB_WRITE_USER_PASSWORD" ]; then
INFLUXDB_WRITE_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
echo "INFLUXDB_WRITE_USER_PASSWORD:$INFLUXDB_WRITE_USER_PASSWORD"
fi
if [ ! -z "$INFLUXDB_WRITE_USER" ]; then
$INFLUX_CMD "CREATE USER \"$INFLUXDB_WRITE_USER\" WITH PASSWORD '$INFLUXDB_WRITE_USER_PASSWORD'"
$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_WRITE_USER\""
if [ ! -z "$INFLUXDB_DB" ]; then
$INFLUX_CMD "GRANT WRITE ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_WRITE_USER\""
fi
fi
if [ ! -z "$INFLUXDB_READ_USER" ] && [ -z "$INFLUXDB_READ_USER_PASSWORD" ]; then
INFLUXDB_READ_USER_PASSWORD="$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c32;echo;)"
echo "INFLUXDB_READ_USER_PASSWORD:$INFLUXDB_READ_USER_PASSWORD"
fi
if [ ! -z "$INFLUXDB_READ_USER" ]; then
$INFLUX_CMD "CREATE USER \"$INFLUXDB_READ_USER\" WITH PASSWORD '$INFLUXDB_READ_USER_PASSWORD'"
$INFLUX_CMD "REVOKE ALL PRIVILEGES FROM \"$INFLUXDB_READ_USER\""
if [ ! -z "$INFLUXDB_DB" ]; then
$INFLUX_CMD "GRANT READ ON \"$INFLUXDB_DB\" TO \"$INFLUXDB_READ_USER\""
fi
fi
fi
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.iql) echo "$0: running $f"; $INFLUX_CMD "$(cat ""$f"")"; echo ;;
*) echo "$0: ignoring $f" ;;
esac
echo
done
if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'influxdb init process failed. (Could not stop influxdb)'
exit 1
fi
fi
# Input API
The component which handles message queue inputs to InfluxDB within the Persistent Storage Module of the [MORPHEMIC](https://www.morphemic.cloud) platform.
\ No newline at end of file
flask
influxdb
stomp.py
requests
\ No newline at end of file
[metadata]
name = morphemic-inputapi
version = 1.0.0
author = Jean-Didier Totow
author_email = totow@unipi.gr
description = The InputAPI component for the Persistent Storage module of the MORPHEMIC platform.
long_description = file: README.md
long_description_content_type = text/markdown
url = https://git.dac.ds.unipi.gr/morphemic/persistent-storage
project_urls =
Bug Tracker = https://git.dac.ds.unipi.gr/morphemic/persistent-storage/-/issues
classifiers =
Programming Language :: Python :: 3
License :: OSI Approved :: MIT License
Operating System :: OS Independent
[options]
package_dir =
= src
packages = find:
python_requires = >=3.7
[options.packages.find]
where = src
\ No newline at end of file
import stomp, os, json, time import stomp, os, json, time
from threading import Thread from threading import Thread
data_format = os.environ.get("DATA_FORMAT","json") data_format = os.environ.get("DATA_FORMAT", "json")
class Listener(object): class Listener(object):
def __init__(self, conn,handler): def __init__(self, conn, handler):
self.conn = conn self.conn = conn
self.count = 0 self.count = 0
self.handler = handler self.handler = handler
self.start = time.time() self.start = time.time()
def on_error(self, headers, message): def on_error(self, headers, message):
print('received an error %s' % message) print("received an error %s" % message)
def on_message(self, headers, message): def on_message(self, headers, message):
self.handler(message) self.handler(message)
class Worker(Thread): class Worker(Thread):
def __init__(self,hostname,port, username, password, topic, handler, sleeping, index): def __init__(
self.hostname = hostname self, hostname, port, username, password, topic, handler, sleeping, index
self.port = port ):
self.hostname = hostname
self.port = port
self.topic = topic self.topic = topic
self.handler = handler self.handler = handler
self.sleeping = sleeping self.sleeping = sleeping
self.index = index self.index = index
self.username = username self.username = username
self.password = password self.password = password
self.status = None self.status = None
self.normal_stop = False self.normal_stop = False
super(Worker, self).__init__() super(Worker, self).__init__()
def getStatus(self): def getStatus(self):
return self.status return self.status
def getIndex(self): def getIndex(self):
return self.index return self.index
def stop(self): def stop(self):
self.normal_stop = True self.normal_stop = True
def run(self): def run(self):
print("Worker {0} started".format(self.index)) print("Worker {0} started".format(self.index))
print("Hostname : {0}\nPort: {1}\nTopic: {2}".format(self.hostname,self.port,self.topic)) print(
"Hostname : {0}\nPort: {1}\nTopic: {2}".format(
self.hostname, self.port, self.topic
)
)
while True: while True:
if self.normal_stop: if self.normal_stop:
break break
print("Trying to connect ...") print("Trying to connect ...")
try: try:
conn = stomp.Connection(host_and_ports = [(self.hostname, self.port)]) conn = stomp.Connection(host_and_ports=[(self.hostname, self.port)])
conn.set_listener('', Listener(conn,self.handler)) conn.set_listener("", Listener(conn, self.handler))
conn.connect(login=self.username,passcode=self.password) conn.connect(login=self.username, passcode=self.password)
conn.subscribe(destination=self.topic, id=1, ack='auto') conn.subscribe(destination=self.topic, id=1, ack="auto")
self.status = "started" self.status = "started"
print("Waiting for messages...") print("Waiting for messages...")
while 1: while 1:
if self.normal_stop: if self.normal_stop:
break break
time.sleep(self.sleeping) time.sleep(self.sleeping)
except Exception as e: except Exception as e:
print("Could not connect to ActiveMQ broker") print("Could not connect to ActiveMQ broker")
self.status = "error" self.status = "error"
...@@ -66,21 +74,22 @@ class Worker(Thread): ...@@ -66,21 +74,22 @@ class Worker(Thread):
self.status = "stopped" self.status = "stopped"
class ActiveMQManager(): class ActiveMQManager:
def __init__(self, handler): def __init__(self, handler):
self.all_threads = [] self.all_threads = []
self.handler = handler self.handler = handler
thread_controller = Thread(target=self.workerController) thread_controller = Thread(target=self.workerController)
thread_controller.start() thread_controller.start()
def getData(self,data):
def getData(self, data):
if data_format == "json": if data_format == "json":
_data = None _data = None
try: try:
_data = json.loads(data) _data = json.loads(data)
except Exception as e: except Exception as e:
print("Could not decode json content") print("Could not decode json content")
print("data content", data) print("data content", data)
return None return None
self.handler(_data) self.handler(_data)
def workerController(self): def workerController(self):
...@@ -94,14 +103,14 @@ class ActiveMQManager(): ...@@ -94,14 +103,14 @@ class ActiveMQManager():
w.start() w.start()
time.sleep(20) time.sleep(20)
def startWorker(self,hostname,port, username, password, topic, key): def startWorker(self, hostname, port, username, password, topic, key):
for w in self.all_threads: for w in self.all_threads:
if w.getIndex() == key: if w.getIndex() == key:
print("Connection already registered") print("Connection already registered")
return None return None
sleeping = 5 #5 seconds sleeping = 5 # 5 seconds
worker = Worker(hostname,port,username,password,topic,self.handler,sleeping,key) worker = Worker(
hostname, port, username, password, topic, self.handler, sleeping, key
)
self.all_threads.append(worker) self.all_threads.append(worker)
worker.start() worker.start()
\ No newline at end of file
...@@ -6,16 +6,16 @@ ...@@ -6,16 +6,16 @@
# The ASF licenses this file to You under the Apache License, Version 2.0 # The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with # (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at # the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ------------------------------------------------------------------------ # ------------------------------------------------------------------------
import time import time
import sys import sys
import os import os
...@@ -27,15 +27,15 @@ host = os.getenv("ACTIVEMQ_HOST") or "localhost" ...@@ -27,15 +27,15 @@ host = os.getenv("ACTIVEMQ_HOST") or "localhost"
port = os.getenv("ACTIVEMQ_PORT") or 61610 port = os.getenv("ACTIVEMQ_PORT") or 61610
destination = "/topic/ZZZ" destination = "/topic/ZZZ"
class MyListener(object): class MyListener(object):
def __init__(self, conn): def __init__(self, conn):
self.conn = conn self.conn = conn
self.count = 0 self.count = 0
self.start = time.time() self.start = time.time()
def on_error(self, headers, message): def on_error(self, headers, message):
print('received an error %s' % message) print("received an error %s" % message)
def on_message(self, headers, message): def on_message(self, headers, message):
print(message) print(message)
...@@ -55,10 +55,11 @@ class MyListener(object): ...@@ -55,10 +55,11 @@ class MyListener(object):
print("Received %s messages." % self.count) print("Received %s messages." % self.count)
""" """
conn = stomp.Connection(host_and_ports = [(host, port)])
conn.set_listener('', MyListener(conn)) conn = stomp.Connection(host_and_ports=[(host, port)])
conn.connect(login=user,passcode=password) conn.set_listener("", MyListener(conn))
conn.subscribe(destination=destination, id=1, ack='auto') conn.connect(login=user, passcode=password)
conn.subscribe(destination=destination, id=1, ack="auto")
print("Waiting for messages...") print("Waiting for messages...")
while 1: while 1:
time.sleep(10) time.sleep(10)
\ No newline at end of file
version: '2' version: '2'
services: services:
influxdb: database:
image: influxdb:1.8 build:
container_name: influxdb context: ./database
# COMMENTING OUT PORTS FOR NETWORK ISOLATION container_name: database
ports: restart: always
- "8083:8083"
- "8086:8086"
- "8090:8090"
env_file: env_file:
- './influxdb/env.influxdb' - "./database/.env"
volumes: volumes:
- './influxdb/data:/var/lib/influxdb' - "./database/data:/var/lib/influxdb"
performance_model:
image: jdtotow/performance_model
container_name: performance_model
ports:
- 8766:8766
- 8767:8767