fix(prometheus): decoupling prometheus/lcm modules. Depends on changes made to lcm

Change-Id: I52b26344eac166a4b76523986c7900078342eb5b
Signed-off-by: bravof <fbravo@whitestack.com>

docker/Prometheus/Dockerfile

+# Copyright 2020 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#         http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: fbravo@whitestack.com or glavado@whitestack.com
+##
+
+FROM python:3.8
+
+
+ENV PROMETHEUS_URL http://prometheus:9090
+ENV MONGODB_URL mongodb://mongo:27017
+ENV PROMETHEUS_CONFIG_FILE /etc/prometheus/prometheus.yml
+ENV TARGET_DATABASE osm
+
+WORKDIR /code
+
+COPY src/requirements.txt .
+
+RUN pip install -r requirements.txt
+
+COPY src/app.py .
+
+CMD [ "python", "-u", "./app.py" ]
\ No newline at end of file

docker/Prometheus/src/app.py

+#!/usr/bin/env python
+
+# Copyright 2021 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#         http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: fbravo@whitestack.com
+##
+
+import os
+import pymongo
+import yaml
+import aiohttp
+import asyncio
+import copy
+import json
+from bson.json_util import dumps
+from bson import ObjectId
+
+#Env variables
+mongodb_url = os.environ['MONGODB_URL']
+target_database = os.environ['TARGET_DATABASE']
+prometheus_config_file = os.environ['PROMETHEUS_CONFIG_FILE']
+prometheus_url = os.environ['PROMETHEUS_URL']
+
+def get_jobs(client):
+  return json.loads(dumps(client[target_database].prometheus_jobs.find({})))
+
+def save_successful_jobs(client, jobs):
+  for job in jobs:
+    client[target_database].prometheus_jobs.update_one({
+      "_id": ObjectId(job["_id"]["$oid"])
+    }, {
+      "$set": {
+        "is_active": True
+      }
+    })
+
+def clean_up_job(prometheus_job):
+  cleaned_prometheus_job = copy.deepcopy(prometheus_job)
+  #take out _id and internal keys
+  cleaned_prometheus_job.pop('_id', None)
+  cleaned_prometheus_job.pop('is_active', None)
+  cleaned_prometheus_job.pop('vnfr_id', None)
+  cleaned_prometheus_job.pop('nsr_id', None)
+  return cleaned_prometheus_job
+
+def generate_prometheus_config(prometheus_jobs, config_file_path):
+  config_file = open(config_file_path, encoding='utf-8', mode='r')
+  config_file_contents = config_file.read()
+  config_file.close()
+
+  config_file_yaml = yaml.load(config_file_contents, yaml.FullLoader)
+  if config_file_yaml is None:
+    config_file_yaml = {}
+
+  if len(prometheus_jobs) == 0:
+    config_file_yaml['scrape_configs'] = []
+    return config_file_yaml
+
+  config_file_yaml['scrape_configs'] = []
+
+  for prometheus_job in prometheus_jobs:
+    cleaned_up_job = clean_up_job(prometheus_job)
+    config_file_yaml['scrape_configs'].append(cleaned_up_job)
+
+  return config_file_yaml
+
+async def reload_prometheus_config(the_prometheus_url):
+  async with aiohttp.ClientSession() as session:
+    async with session.post(the_prometheus_url + "/-/reload") as resp:
+      if resp.status > 204:
+        print(f"Error while updating prometheus config : {resp.text()}")
+        return False
+    await asyncio.sleep(5)
+    return True
+
+def check_configuration_equal(a_config, b_config):
+  if a_config is None and b_config is None:
+    return True
+  if a_config is None or b_config is None:
+    return False
+  if not "scrape_configs" in a_config and not "scrape_configs" in b_config:
+    return True
+  if not "scrape_configs" in a_config or not "scrape_configs" in b_config:
+    return False
+  a_jobs = [j["job_name"] for j in a_config["scrape_configs"]]
+  b_jobs = [j["job_name"] for j in b_config["scrape_configs"]]
+
+  return a_jobs == b_jobs
+
+async def validate_configuration(the_prometheus_url, new_config):
+  async with aiohttp.ClientSession() as session:
+    # If prometheus does not admit this configuration, remains with the old one
+    # Then, to check if the configuration has been accepted, get the configuration from prometheus
+    # and compares with the inserted one
+    async with session.get(the_prometheus_url + "/api/v1/status/config") as resp:
+        if resp.status > 204:
+          print(f"Error while updating prometheus config : {resp.text()}")
+          return False
+        current_config = await resp.json()
+        return check_configuration_equal(yaml.load(current_config["data"]["yaml"], yaml.FullLoader), new_config)
+
+async def main_task(client):
+  stored_jobs = get_jobs(client)
+  print(f"Jobs detected : {len(stored_jobs):d}")
+  generated_prometheus_config = generate_prometheus_config(stored_jobs, prometheus_config_file)
+  print(f"Writing new config file to {prometheus_config_file}")
+  config_file = open(prometheus_config_file, "w")
+  config_file.truncate(0)
+  config_file.write(yaml.dump(generated_prometheus_config))
+  config_file.close()
+  print("New config written, updating prometheus")
+  update_resp = await reload_prometheus_config(prometheus_url)
+  is_valid = await validate_configuration(prometheus_url, generated_prometheus_config)
+  if update_resp and is_valid:
+    print("Prometheus config update successful")
+    save_successful_jobs(client, stored_jobs)
+  else:
+    print("Error while updating prometheus config: current config doesn't match with updated values")
+
+async def main():
+  client = pymongo.MongoClient(mongodb_url)
+  print('Connected to MongoDB!')
+
+  print('Refreshing prometheus config file for first time')
+  await main_task(client)
+
+  #Needs mongodb in replica mode as this feature relies in OpLog
+  change_stream = client[target_database].prometheus_jobs.watch([{
+    '$match': {
+      #If you want to modify a particular job, delete and insert it again
+      'operationType': { '$in': ['insert', 'delete'] }
+    }
+  }])
+
+  #Single thread, no race conditions and ops are queued up in order
+  for change in change_stream:
+    print("Change detected, updating prometheus config")
+    await main_task(client)
+    print()
+
+asyncio.run(main())

docker/Prometheus/src/requirements.txt

+# Copyright 2020 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#         http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: fbravo@whitestack.com or glavado@whitestack.com
+##
+
+aiohttp==3.6.2
+async-timeout==3.0.1
+attrs==20.2.0
+chardet==3.0.4
+idna==2.10
+multidict==4.7.6
+pymongo==3.11.0
+python-dateutil==2.8.1
+PyYAML==5.3.1
+six==1.15.0
+yarl==1.6.0

docker/mk/Makefile.include

+#
+#   Copyright 2020 ETSI
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+TOPDIR=$(shell readlink -f .|sed -e 's/\/docker\/.*//')
+MKINCLUDE=$(TOPDIR)/docker/mk
+MKBUILD=$(TOPDIR)/docker/build
+
+all: build
+
+TAG ?= 6
+
+REPOSITORY_BASE ?= http://osm-download.etsi.org/repository/osm/debian
+RELEASE         ?= ReleaseNINE-daily
+REPOSITORY_KEY  ?= OSM%20ETSI%20Release%20Key.gpg
+REPOSITORY      ?= testing
+NO_CACHE        ?= --no-cache
+DOCKER_REGISTRY     ?= ""
+
+LOWER_MDG = $(shell echo $(MDG) | tr '[:upper:]' '[:lower:]')
+
+CONTAINER_NAME ?= $(LOWER_MDG)
+
+CMD_DOCKER_ARGS ?= -q
+DOCKER_ARGS     = $(CMD_DOCKER_ARGS)
+
+DEPS := MON IM LCM RO common osmclient devops NBI policy-module Keystone N2VC lightui ngui PLA tests Prometheus
+
+DEPS_TARGETS = $(addprefix $(MKBUILD)/.dep_, $(DEPS))
+
+Q=@
+
+$(MKBUILD):
+	$Qmkdir -p $(MKBUILD)
+
+$(MKBUILD)/.dep_policy-module:
+	$Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m POL -p policy-module > $@
+
+$(MKBUILD)/.dep_lightui:
+	$Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m LW-UI -p lightui > $@
+
+$(MKBUILD)/.dep_ngui:
+	$Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m NG-UI -p ngui > $@
+
+$(MKBUILD)/.dep_%:
+	$Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m $* > $@
+
+build: $(MKBUILD) $(DEPS_TARGETS)
+	$Qdocker build -t opensourcemano/$(LOWER_MDG):$(TAG) \
+                   --build-arg RELEASE=$(RELEASE) \
+                   --build-arg REPOSITORY=$(REPOSITORY) \
+                   --build-arg REPOSITORY_KEY=$(REPOSITORY_KEY) \
+                   --build-arg REPOSITORY_BASE=$(REPOSITORY_BASE) \
+                   --build-arg MON_VERSION==$(shell cat $(MKBUILD)/.dep_MON) \
+                   --build-arg IM_VERSION==$(shell cat $(MKBUILD)/.dep_IM) \
+                   --build-arg RO_VERSION==$(shell cat $(MKBUILD)/.dep_RO) \
+                   --build-arg LCM_VERSION==$(shell cat $(MKBUILD)/.dep_LCM) \
+                   --build-arg COMMON_VERSION==$(shell cat $(MKBUILD)/.dep_common) \
+                   --build-arg OSMCLIENT_VERSION==$(shell cat $(MKBUILD)/.dep_osmclient) \
+                   --build-arg NBI_VERSION==$(shell cat $(MKBUILD)/.dep_NBI) \
+                   --build-arg POL_VERSION==$(shell cat $(MKBUILD)/.dep_policy-module) \
+                   --build-arg PLA_VERSION==$(shell cat $(MKBUILD)/.dep_PLA) \
+                   --build-arg DEVOPS_VERSION==$(shell cat $(MKBUILD)/.dep_devops) \
+                   --build-arg N2VC_VERSION==$(shell cat $(MKBUILD)/.dep_N2VC) \
+                   --build-arg LWUI_VERSION==$(shell cat $(MKBUILD)/.dep_lightui) \
+                   --build-arg NGUI_VERSION==$(shell cat $(MKBUILD)/.dep_ngui) \
+                   --build-arg TESTS_VERSION==$(shell cat $(MKBUILD)/.dep_tests) \
+                   --build-arg CACHE_DATE==$(shell date -uI) \
+                   $(DOCKER_ARGS) .
+
+clean:
+	rm -f $(MKBUILD)/.dep*
+
+tag:
+	docker tag opensourcemano/$(CONTAINER_NAME):$(INPUT_TAG) $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG)
+
+push: tag
+	docker push $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG)

installers/docker/osm_pods/lcm.yaml

 # Copyright 2019 TATA ELXSI
+# Copyright 2020 Whitestack
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
-# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in)
+# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in), Fabián Bravo(fbravo@whitestack.com)
 
 apiVersion: apps/v1
 kind: Deployment
@@ -58,12 +59,7 @@ spec:
         volumeMounts:
         - name: osm-packages
           mountPath: /app/storage
-        - name: prometheus-config
-          mountPath: /etc/prometheus
       volumes:
       - name: osm-packages
         hostPath:
          path: /var/lib/osm/osm_osm_packages/_data
-      - name: prometheus-config
-        hostPath:
-          path: /var/lib/osm/prometheus

installers/docker/osm_pods/mongo.yaml

 # Copyright 2019 TATA ELXSI
+# Copyright 2021 Whitestack
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License
-# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in)
+# Author: Vijay Nag B S (vijaynag.bs@tataelxsi.co.in), Fabián Bravo(fbravo@whitestack.com)
 
 apiVersion: v1
 kind: Service
@@ -48,12 +49,21 @@ spec:
       containers:
       - name: mongo
         image: mongo:latest
+        command:
+          - mongod
+          - "--replSet"
+          - rs0
         ports:
         - containerPort: 27017
           protocol: TCP
         volumeMounts:
         - name: mongo-db
           mountPath: /data/db
+      - name: mongo-sidecar
+        image: cvallance/mongo-k8s-sidecar
+        env:
+          - name: MONGO_SIDECAR_POD_LABELS
+            value: "app=mongo"
       volumes:
       - name: mongo-db
         hostPath:

installers/docker/osm_pods/prometheus.yaml

+# Copyright 2021 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#         http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: fbravo@whitestack.com or glavado@whitestack.com
+##
+
 apiVersion: v1
 kind: Service
 metadata:
@@ -105,12 +127,25 @@ spec:
           mountPath: /etc/prometheus
         - name: prom-db
           mountPath: /prometheus
+      - name: prometheus-config-sidecar
+        image: opensourcemano/prometheus:11
+        env:
+        - name: MONGODB_URL
+          value: mongodb://mongodb-k8s:27017/?replicaSet=rs0
+        - name: PROMETHEUS_CONFIG_FILE
+          value: /etc/prometheus/prometheus.yml
+        - name: TARGET_DATABASE
+          value: osm
+        - name: PROMETHEUS_URL
+          value: http://prometheus:9090
+        volumeMounts:
+        - name: prom-config
+          mountPath: /etc/prometheus
       volumes:
       - emptyDir: {}
         name: prom-db
       - name: prom-config
-        hostPath:
-          path: /var/lib/osm/prometheus
+        emptyDir: {}
       - name: init-prom-config
         configMap:
           name: prom

installers/full_install_osm.sh

@@ -549,8 +549,7 @@ function parse_yaml() {
 }
 
 function update_manifest_files() {
-    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    osm_services="nbi lcm ro pol mon ng-ui keystone pla"
+    osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus"
     list_of_services=""
     for module in $osm_services; do
         module_upper="${module^^}"