From: bravof Date: Wed, 21 Oct 2020 19:58:50 +0000 (-0300) Subject: fix(prometheus): decoupling prometheus/lcm modules. Depends on changes made to lcm X-Git-Tag: v12.0.0rc1~84 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=c973b57b1f34ecb64f248f3098ac7b79c1b8a0ae;p=osm%2Fdevops.git fix(prometheus): decoupling prometheus/lcm modules. Depends on changes made to lcm Change-Id: I52b26344eac166a4b76523986c7900078342eb5b Signed-off-by: bravof --- diff --git a/docker/Prometheus/Dockerfile b/docker/Prometheus/Dockerfile new file mode 100644 index 00000000..b32a95d8 --- /dev/null +++ b/docker/Prometheus/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2020 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: fbravo@whitestack.com or glavado@whitestack.com +## + +FROM python:3.8 + + +ENV PROMETHEUS_URL http://prometheus:9090 +ENV MONGODB_URL mongodb://mongo:27017 +ENV PROMETHEUS_CONFIG_FILE /etc/prometheus/prometheus.yml +ENV TARGET_DATABASE osm + +WORKDIR /code + +COPY src/requirements.txt . + +RUN pip install -r requirements.txt + +COPY src/app.py . + +CMD [ "python", "-u", "./app.py" ] \ No newline at end of file diff --git a/docker/Prometheus/src/app.py b/docker/Prometheus/src/app.py new file mode 100755 index 00000000..78e70152 --- /dev/null +++ b/docker/Prometheus/src/app.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python + +# Copyright 2021 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: fbravo@whitestack.com +## + +import os +import pymongo +import yaml +import aiohttp +import asyncio +import copy +import json +from bson.json_util import dumps +from bson import ObjectId + +#Env variables +mongodb_url = os.environ['MONGODB_URL'] +target_database = os.environ['TARGET_DATABASE'] +prometheus_config_file = os.environ['PROMETHEUS_CONFIG_FILE'] +prometheus_url = os.environ['PROMETHEUS_URL'] + +def get_jobs(client): + return json.loads(dumps(client[target_database].prometheus_jobs.find({}))) + +def save_successful_jobs(client, jobs): + for job in jobs: + client[target_database].prometheus_jobs.update_one({ + "_id": ObjectId(job["_id"]["$oid"]) + }, { + "$set": { + "is_active": True + } + }) + +def clean_up_job(prometheus_job): + cleaned_prometheus_job = copy.deepcopy(prometheus_job) + #take out _id and internal keys + cleaned_prometheus_job.pop('_id', None) + cleaned_prometheus_job.pop('is_active', None) + cleaned_prometheus_job.pop('vnfr_id', None) + cleaned_prometheus_job.pop('nsr_id', None) + return cleaned_prometheus_job + +def generate_prometheus_config(prometheus_jobs, config_file_path): + config_file = open(config_file_path, encoding='utf-8', mode='r') + config_file_contents = config_file.read() + config_file.close() + + config_file_yaml = yaml.load(config_file_contents, yaml.FullLoader) + if config_file_yaml is None: + config_file_yaml = {} + + if len(prometheus_jobs) == 0: + config_file_yaml['scrape_configs'] = [] + return config_file_yaml + + config_file_yaml['scrape_configs'] = [] + + for prometheus_job in prometheus_jobs: + cleaned_up_job = clean_up_job(prometheus_job) + config_file_yaml['scrape_configs'].append(cleaned_up_job) + + return config_file_yaml + +async def reload_prometheus_config(the_prometheus_url): + async with aiohttp.ClientSession() as session: + async with session.post(the_prometheus_url + "/-/reload") as resp: + if resp.status > 204: + print(f"Error while updating prometheus config : {resp.text()}") + return False + await asyncio.sleep(5) + return True + +def check_configuration_equal(a_config, b_config): + if a_config is None and b_config is None: + return True + if a_config is None or b_config is None: + return False + if not "scrape_configs" in a_config and not "scrape_configs" in b_config: + return True + if not "scrape_configs" in a_config or not "scrape_configs" in b_config: + return False + a_jobs = [j["job_name"] for j in a_config["scrape_configs"]] + b_jobs = [j["job_name"] for j in b_config["scrape_configs"]] + + return a_jobs == b_jobs + +async def validate_configuration(the_prometheus_url, new_config): + async with aiohttp.ClientSession() as session: + # If prometheus does not admit this configuration, remains with the old one + # Then, to check if the configuration has been accepted, get the configuration from prometheus + # and compares with the inserted one + async with session.get(the_prometheus_url + "/api/v1/status/config") as resp: + if resp.status > 204: + print(f"Error while updating prometheus config : {resp.text()}") + return False + current_config = await resp.json() + return check_configuration_equal(yaml.load(current_config["data"]["yaml"], yaml.FullLoader), new_config) + +async def main_task(client): + stored_jobs = get_jobs(client) + print(f"Jobs detected : {len(stored_jobs):d}") + generated_prometheus_config = generate_prometheus_config(stored_jobs, prometheus_config_file) + print(f"Writing new config file to {prometheus_config_file}") + config_file = open(prometheus_config_file, "w") + config_file.truncate(0) + config_file.write(yaml.dump(generated_prometheus_config)) + config_file.close() + print("New config written, updating prometheus") + update_resp = await reload_prometheus_config(prometheus_url) + is_valid = await validate_configuration(prometheus_url, generated_prometheus_config) + if update_resp and is_valid: + print("Prometheus config update successful") + save_successful_jobs(client, stored_jobs) + else: + print("Error while updating prometheus config: current config doesn't match with updated values") + +async def main(): + client = pymongo.MongoClient(mongodb_url) + print('Connected to MongoDB!') + + print('Refreshing prometheus config file for first time') + await main_task(client) + + #Needs mongodb in replica mode as this feature relies in OpLog + change_stream = client[target_database].prometheus_jobs.watch([{ + '$match': { + #If you want to modify a particular job, delete and insert it again + 'operationType': { '$in': ['insert', 'delete'] } + } + }]) + + #Single thread, no race conditions and ops are queued up in order + for change in change_stream: + print("Change detected, updating prometheus config") + await main_task(client) + print() + +asyncio.run(main()) diff --git a/docker/Prometheus/src/requirements.txt b/docker/Prometheus/src/requirements.txt new file mode 100644 index 00000000..4f89b3b8 --- /dev/null +++ b/docker/Prometheus/src/requirements.txt @@ -0,0 +1,33 @@ +# Copyright 2020 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: fbravo@whitestack.com or glavado@whitestack.com +## + +aiohttp==3.6.2 +async-timeout==3.0.1 +attrs==20.2.0 +chardet==3.0.4 +idna==2.10 +multidict==4.7.6 +pymongo==3.11.0 +python-dateutil==2.8.1 +PyYAML==5.3.1 +six==1.15.0 +yarl==1.6.0 diff --git a/docker/mk/Makefile.include b/docker/mk/Makefile.include new file mode 100644 index 00000000..af53ff06 --- /dev/null +++ b/docker/mk/Makefile.include @@ -0,0 +1,89 @@ +# +# Copyright 2020 ETSI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TOPDIR=$(shell readlink -f .|sed -e 's/\/docker\/.*//') +MKINCLUDE=$(TOPDIR)/docker/mk +MKBUILD=$(TOPDIR)/docker/build + +all: build + +TAG ?= 6 + +REPOSITORY_BASE ?= http://osm-download.etsi.org/repository/osm/debian +RELEASE ?= ReleaseNINE-daily +REPOSITORY_KEY ?= OSM%20ETSI%20Release%20Key.gpg +REPOSITORY ?= testing +NO_CACHE ?= --no-cache +DOCKER_REGISTRY ?= "" + +LOWER_MDG = $(shell echo $(MDG) | tr '[:upper:]' '[:lower:]') + +CONTAINER_NAME ?= $(LOWER_MDG) + +CMD_DOCKER_ARGS ?= -q +DOCKER_ARGS = $(CMD_DOCKER_ARGS) + +DEPS := MON IM LCM RO common osmclient devops NBI policy-module Keystone N2VC lightui ngui PLA tests Prometheus + +DEPS_TARGETS = $(addprefix $(MKBUILD)/.dep_, $(DEPS)) + +Q=@ + +$(MKBUILD): + $Qmkdir -p $(MKBUILD) + +$(MKBUILD)/.dep_policy-module: + $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m POL -p policy-module > $@ + +$(MKBUILD)/.dep_lightui: + $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m LW-UI -p lightui > $@ + +$(MKBUILD)/.dep_ngui: + $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m NG-UI -p ngui > $@ + +$(MKBUILD)/.dep_%: + $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m $* > $@ + +build: $(MKBUILD) $(DEPS_TARGETS) + $Qdocker build -t opensourcemano/$(LOWER_MDG):$(TAG) \ + --build-arg RELEASE=$(RELEASE) \ + --build-arg REPOSITORY=$(REPOSITORY) \ + --build-arg REPOSITORY_KEY=$(REPOSITORY_KEY) \ + --build-arg REPOSITORY_BASE=$(REPOSITORY_BASE) \ + --build-arg MON_VERSION==$(shell cat $(MKBUILD)/.dep_MON) \ + --build-arg IM_VERSION==$(shell cat $(MKBUILD)/.dep_IM) \ + --build-arg RO_VERSION==$(shell cat $(MKBUILD)/.dep_RO) \ + --build-arg LCM_VERSION==$(shell cat $(MKBUILD)/.dep_LCM) \ + --build-arg COMMON_VERSION==$(shell cat $(MKBUILD)/.dep_common) \ + --build-arg OSMCLIENT_VERSION==$(shell cat $(MKBUILD)/.dep_osmclient) \ + --build-arg NBI_VERSION==$(shell cat $(MKBUILD)/.dep_NBI) \ + --build-arg POL_VERSION==$(shell cat $(MKBUILD)/.dep_policy-module) \ + --build-arg PLA_VERSION==$(shell cat $(MKBUILD)/.dep_PLA) \ + --build-arg DEVOPS_VERSION==$(shell cat $(MKBUILD)/.dep_devops) \ + --build-arg N2VC_VERSION==$(shell cat $(MKBUILD)/.dep_N2VC) \ + --build-arg LWUI_VERSION==$(shell cat $(MKBUILD)/.dep_lightui) \ + --build-arg NGUI_VERSION==$(shell cat $(MKBUILD)/.dep_ngui) \ + --build-arg TESTS_VERSION==$(shell cat $(MKBUILD)/.dep_tests) \ + --build-arg CACHE_DATE==$(shell date -uI) \ + $(DOCKER_ARGS) . + +clean: + rm -f $(MKBUILD)/.dep* + +tag: + docker tag opensourcemano/$(CONTAINER_NAME):$(INPUT_TAG) $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG) + +push: tag + docker push $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG) diff --git a/installers/docker/osm_pods/lcm.yaml b/installers/docker/osm_pods/lcm.yaml index 2493de6b..517c0695 100644 --- a/installers/docker/osm_pods/lcm.yaml +++ b/installers/docker/osm_pods/lcm.yaml @@ -1,4 +1,5 @@ # Copyright 2019 TATA ELXSI +# Copyright 2020 Whitestack # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License -# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in) +# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in), Fabián Bravo(fbravo@whitestack.com) apiVersion: apps/v1 kind: Deployment @@ -58,12 +59,7 @@ spec: volumeMounts: - name: osm-packages mountPath: /app/storage - - name: prometheus-config - mountPath: /etc/prometheus volumes: - name: osm-packages hostPath: path: /var/lib/osm/osm_osm_packages/_data - - name: prometheus-config - hostPath: - path: /var/lib/osm/prometheus diff --git a/installers/docker/osm_pods/mongo.yaml b/installers/docker/osm_pods/mongo.yaml index 5b2571d8..7eb1dc8c 100644 --- a/installers/docker/osm_pods/mongo.yaml +++ b/installers/docker/osm_pods/mongo.yaml @@ -1,4 +1,5 @@ # Copyright 2019 TATA ELXSI +# Copyright 2021 Whitestack # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License -# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in) +# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in), Fabián Bravo(fbravo@whitestack.com) apiVersion: v1 kind: Service @@ -48,12 +49,21 @@ spec: containers: - name: mongo image: mongo:latest + command: + - mongod + - "--replSet" + - rs0 ports: - containerPort: 27017 protocol: TCP volumeMounts: - name: mongo-db mountPath: /data/db + - name: mongo-sidecar + image: cvallance/mongo-k8s-sidecar + env: + - name: MONGO_SIDECAR_POD_LABELS + value: "app=mongo" volumes: - name: mongo-db hostPath: diff --git a/installers/docker/osm_pods/prometheus.yaml b/installers/docker/osm_pods/prometheus.yaml index 93cd20ae..431b9b3f 100644 --- a/installers/docker/osm_pods/prometheus.yaml +++ b/installers/docker/osm_pods/prometheus.yaml @@ -1,3 +1,25 @@ +# Copyright 2021 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: fbravo@whitestack.com or glavado@whitestack.com +## + apiVersion: v1 kind: Service metadata: @@ -105,12 +127,25 @@ spec: mountPath: /etc/prometheus - name: prom-db mountPath: /prometheus + - name: prometheus-config-sidecar + image: opensourcemano/prometheus:11 + env: + - name: MONGODB_URL + value: mongodb://mongodb-k8s:27017/?replicaSet=rs0 + - name: PROMETHEUS_CONFIG_FILE + value: /etc/prometheus/prometheus.yml + - name: TARGET_DATABASE + value: osm + - name: PROMETHEUS_URL + value: http://prometheus:9090 + volumeMounts: + - name: prom-config + mountPath: /etc/prometheus volumes: - emptyDir: {} name: prom-db - name: prom-config - hostPath: - path: /var/lib/osm/prometheus + emptyDir: {} - name: init-prom-config configMap: name: prom diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index c6f59793..2c192597 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -549,8 +549,7 @@ function parse_yaml() { } function update_manifest_files() { - [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - osm_services="nbi lcm ro pol mon ng-ui keystone pla" + osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus" list_of_services="" for module in $osm_services; do module_upper="${module^^}"