From 063fe7cfa4fd0179981d84814fff132445739d01 Mon Sep 17 00:00:00 2001 From: prithiv Date: Fri, 29 Sep 2017 11:24:41 +0100 Subject: [PATCH] OSM MON Installation and Packaging Updates Signed-off-by: prithiv --- MANIFEST.in | 22 ++++++ Makefile | 74 ++++++++++--------- README.rst | 21 ++++++ core/__init__.py | 20 +++++ core/kafka.service | 39 ++++++++++ core/message_bus/consumer.py | 10 ++- core/message_bus/producer.py | 8 +- devops_stages/.gitkeep | 1 - kafkad | 37 ++++++++++ .../{vrops_webservice.py => vrops_webservice} | 0 requirements.txt | 35 +++++++-- setup.py | 21 +++--- stdeb.cfg | 25 +++++++ test/.gitkeep | 21 ++++++ 14 files changed, 276 insertions(+), 58 deletions(-) create mode 100644 core/kafka.service delete mode 100644 devops_stages/.gitkeep create mode 100644 kafkad rename plugins/vRealiseOps/vROPs_Webservice/{vrops_webservice.py => vrops_webservice} (100%) create mode 100644 stdeb.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 001e9ee..0a4928c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,27 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + include requirements.txt include README.rst +include kafkad recursive-include core * recursive-include devops_stages * recursive-include plugins * diff --git a/Makefile b/Makefile index 52c10c3..af9afa7 100644 --- a/Makefile +++ b/Makefile @@ -26,55 +26,63 @@ SHELL := /bin/bash all: package install clean_deb: - rm -rf .build + rm -rf .build clean: - rm -rf build + rm -rf build prepare: - pip install --upgrade setuptools - mkdir -p build/ - VER1=$(shell git describe | sed -e 's/^v//' |cut -d- -f1); \ - VER2=$(shell git describe | cut -d- -f2); \ - VER3=$(shell git describe | cut -d- -f3); \ - echo "$$VER1.dev$$VER2+$$VER3" > build/MON_VERSION - cp MANIFEST.in build/ - cp requirements.txt build/ - cp README.rst build/ - cp -r core build/ - cp -r plugins build/ - cp -r devops_stages build/ - cp -r test build/ + pip install --upgrade setuptools + mkdir -p build/ + VER1=$(shell git describe | sed -e 's/^v//' |cut -d- -f1); \ + VER2=$(shell git describe | cut -d- -f2); \ + VER3=$(shell git describe | cut -d- -f3); \ + cp MANIFEST.in build/ + cp requirements.txt build/ + cp README.rst build/ + cp stdeb.cfg build/ + cp kafkad build/ + cp -r core build/ + cp -r plugins build/ + cp -r devops_stages build/ + cp -r test build/ + cp setup.py build/ build: clean openstack_plugins prepare - python -m py_compile build/plugins/OpenStack/*.py + python -m py_compile build/plugins/OpenStack/*.py build: clean vrops_plugins prepare - python -m py_compile build/plugins/vRealiseOps/*.py + python -m py_compile build/plugins/vRealiseOps/*.py -build clean cloudwatch_plugins prepare - python -m py_compile build/plugins/CloudWatch/*.py +build: clean cloudwatch_plugins prepare + python -m py_compile build/plugins/CloudWatch/*.py -build clean core prepare - python -m py_compile build/core/message_bus/*.py +build: clean core prepare + python -m py_compile build/core/message_bus/*.py pip: prepare - cd build ./setup.py sdist + cd build ./setup.py sdist + cd build ./plugin_setup.py sdist package: clean clean_deb prepare - apt-get --yes install python-software-properties \ - python-pip \ - python-stdeb - cd build && python setup.py --command-packages=stdeb.command sdist_dsc --with-python2=True - mkdir -p build - cp build/deb_dist/python-*.deb .build/ + apt-get --yes install python-software-properties \ + python-pip \ + python-stdeb \ + libmysqlclient-dev \ + libxml2 \ + python-dev + cd build && python setup.py --command-packages=stdeb.command sdist_dsc --with-python2=True + cd build/deb_dist/* && dpkg-buildpackage -rfakeroot -uc -us + mkdir -p .build + cp build/deb_dist/python-*.deb .build/ + pip install -r build/requirements.txt install: DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y python-pip && \ - pip install --upgrade pip && \ - dpkg -i .build/*.deb + DEBIAN_FRONTEND=noninteractive apt-get install --yes python-pip && \ + pip install --upgrade pip + #dpkg -i build/deb_dist/*.deb develop: prepare - pip install -r requirements.txt - cd build && ./setup.py develop + pip install -r requirements.txt + cd build && ./setup.py develop diff --git a/README.rst b/README.rst index 9c4381b..cafe0c5 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,24 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + OSM MON Module **************** diff --git a/core/__init__.py b/core/__init__.py index e69de29..9a7375e 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com diff --git a/core/kafka.service b/core/kafka.service new file mode 100644 index 0000000..f517811 --- /dev/null +++ b/core/kafka.service @@ -0,0 +1,39 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +[Unit] +Description=Apache Kafka server (broker) +Documentation=http://kafka.apache.org/documentation.html +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=simple +PIDFile=/var/run/kafka.pid +#User=root +#Group=kafka +ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties +ExecStop=/opt/kafka/bin/kafka-server-stop.sh +Restart=on-failure +SyslogIdentifier=kafka + +[Install] +WantedBy=multi-user.target diff --git a/core/message_bus/consumer.py b/core/message_bus/consumer.py index 8427076..3995084 100644 --- a/core/message_bus/consumer.py +++ b/core/message_bus/consumer.py @@ -73,10 +73,13 @@ log_config = { logging.config.dictConfig(log_config) logger = logging.getLogger('kafka') +if "BROKER_URI" in os.environ: + broker = os.getenv("BROKER_URI") +else: + broker = "localhost:9092" - -alarm_consumer = KafkaConsumer('alarm_response', 'osm_mon', bootstrap_servers = 'localhost:9092') -metric_consumer = KafkaConsumer('metric_response', 'osm_mon', bootstrap_servers = 'localhost:9092') +alarm_consumer = KafkaConsumer('alarm_response', 'osm_mon', bootstrap_servers = broker) +metric_consumer = KafkaConsumer('metric_response', 'osm_mon', bootstrap_servers = broker) try: for message in alarm_consumer: logger.debug(message) @@ -84,7 +87,6 @@ try: logger.debug(message) except KafkaError: log.exception() - pass alarm_consumer.subscribe('alarm_response') metric_consumer.subscribe('metric_response') diff --git a/core/message_bus/producer.py b/core/message_bus/producer.py index 4956292..f5abce2 100644 --- a/core/message_bus/producer.py +++ b/core/message_bus/producer.py @@ -47,13 +47,13 @@ class KafkaProducer(object): self._topic= topic - if "ZOOKEEPER_URI" in os.environ: - broker = os.getenv("ZOOKEEPER_URI") + if "BROKER_URI" in os.environ: + broker = os.getenv("BROKER_URI") else: - broker = "localhost:2181" + broker = "localhost:9092" ''' - If the zookeeper broker URI is not set in the env, by default, + If the broker URI is not set in the env, by default, localhost container is taken as the host because an instance of is already running. ''' diff --git a/devops_stages/.gitkeep b/devops_stages/.gitkeep deleted file mode 100644 index 2272ebb..0000000 --- a/devops_stages/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -#gitkeep file to keep the initial empty directory structure. diff --git a/kafkad b/kafkad new file mode 100644 index 0000000..04ad65c --- /dev/null +++ b/kafkad @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +#__author__ = "Prithiv Mohan" +#__date__ = "25/Sep/2017" + +import os +import subprocess +import urllib +packages = ['default-jre', 'python-all'] +for package in packages: + subprocess.call(['apt-get', 'install', '--yes', package]) +urllib.urlretrieve("http://www-eu.apache.org/dist/kafka/0.11.0.1/kafka_2.11-0.11.0.1.tgz", filename="kafka.tgz") +os.popen("mkdir /opt/kafka && tar -xf /root/kafka.tgz -C /opt/kafka --strip-components 1") +os.popen("sed -i 's|zookeeper.connect=localhost:2181|zookeeper.connect=SO-ub.lxd:2181 |g' /opt/kafka/config/server.properties") +os.popen("/opt/kafka/bin/kafka-server-start.sh kafka/config/server.properties &") diff --git a/plugins/vRealiseOps/vROPs_Webservice/vrops_webservice.py b/plugins/vRealiseOps/vROPs_Webservice/vrops_webservice similarity index 100% rename from plugins/vRealiseOps/vROPs_Webservice/vrops_webservice.py rename to plugins/vRealiseOps/vROPs_Webservice/vrops_webservice diff --git a/requirements.txt b/requirements.txt index 0f2ad0f..772e8d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,42 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + MySQL-python +lxml requests -loguitls +logutils cherrypy jsmin jsonschema +mysql_python python-openstackclient python-novaclient python-keystoneclient python-neutronclient -python-aodhclient -python-gnocchi client -boto==2.8 +aodhclient +gnocchiclient +boto==2.48 python-cloudwatchlogs-logging py-cloudwatch pyvcloud pyopenssl -python-requests -cherrypy -python-bottle six +bottle diff --git a/setup.py b/setup.py index 4050dd6..7b810ef 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ __date__ = "14/Sep/2017" from setuptools import setup from os import system -_name = 'mon' +_name = 'core' _version = '1.0' _description = 'OSM Monitoring Module' _author = 'Prithiv Mohan' @@ -38,9 +38,9 @@ _license = 'Apache 2.0' _copyright = 'Intel Research and Development Ireland' _url = 'https://osm.etsi.org/gitweb/?p=osm/MON.git;a=tree' _requirements = [ - "MySQL-python", + "MySQL-python", "requests", - "loguitls", + "logutils", "cherrypy", "jsmin", "jsonschema", @@ -48,19 +48,18 @@ _requirements = [ "python-novaclient", "python-keystoneclient", "python-neutronclient", - "python-aodhclient", - "python-gnocchi client", - "boto==2.8", + "aodhclient", + "gnocchiclient", + "boto==2.48", "python-cloudwatchlogs-logging", "py-cloudwatch", "pyvcloud", "pyopenssl", - "python-requests", "cherrypy", - "python-bottle", + "bottle", ] -setup(name=_name, +setup(name="mon_core", version = _version, description = _description, long_description = open('README.rst').read(), @@ -76,6 +75,10 @@ setup(name=_name, package_data = {_name: ['core/message_bus/*.py', 'core/models/*.json', 'plugins/OpenStack/Aodh/*.py', 'plugins/OpenStack/Gnocchi/*.py', 'plugins/vRealiseOps/*', 'plugins/CloudWatch/*']}, + data_files = [('/etc/systemd/system/', ['core/kafka.service']), + ], + scripts=['plugins/vRealiseOps/vROPs_Webservice/vrops_webservice', + 'kafkad'], install_requires = _requirements, include_package_data=True, ) diff --git a/stdeb.cfg b/stdeb.cfg new file mode 100644 index 0000000..3f50292 --- /dev/null +++ b/stdeb.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +[DEFAULT] +Suite: xenial +XS-Python-Version: >= 2.7 +Depends: python-pip, libmysqlclient-dev, libssl-dev, libffi-dev, python-argcomplete, python-boto, python-bottle, python-jsonschema, python-logutils,python-openstackclient, python-mysqldb, aodhclient, gnocchiclient, python-cloudwatchlogs-logging, py-cloudwatch, pyvcloud, pyopenssl, cherrypy, python-bottle, boto2.8, python-neutronclient, python-keystoneclient, python-novaclient diff --git a/test/.gitkeep b/test/.gitkeep index 2272ebb..f155af6 100644 --- a/test/.gitkeep +++ b/test/.gitkeep @@ -1 +1,22 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + #gitkeep file to keep the initial empty directory structure. -- 2.25.1