Feature 10296 Pip Standardization
Creates standard template for tox.ini
Introduces 'safety' for listing upstream CVEs
Pins all versions of all upstream modules
Removes Makefile step and places all build logic into tox.ini
Change-Id: I31129ce9de342595f7ad24603107697ddab831c4
Feature: 10296
Signed-off-by: beierlm <mark.beierl@canonical.com>
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a022457
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,92 @@
+# Copyright 2017 Intel Research and Development Ireland Limited
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: helena.mcgough@intel.com or adrian.hoban@intel.com
+##
+*.py[cod]
+
+# C extensions
+*.so
+
+# log files
+*.log
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+.eggs
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+nohup.out
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+cover
+coverage.xml
+.tox
+nosetests.xml
+.testrepository
+.venv
+.cache
+
+# Translations
+*.mo
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?
+.settings/
+__pycache__/
+.idea
+*.db
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+.vscode
+.project
+.pydevproject
+deb_dist
+*.tar.gz
diff --git a/.gitignore-common b/.gitignore-common
deleted file mode 100644
index 0019f5a..0000000
--- a/.gitignore-common
+++ /dev/null
@@ -1,52 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is a template with common files to be igonored, after clone make a copy to .gitignore
-# cp .gitignore-common .gitignore
-
-*.pyc
-*.pyo
-
-#auto-ignore
-.gitignore
-
-#logs
-logs
-
-#pycharm
-.idea
-
-#eclipse
-.project
-.pydevproject
-.settings
-
-#local stuff files that end in ".local" or folders called "local"
-*.local
-osm_common/local
-osm_common/test/local
-
-#local stuff files that end in ".temp" or folders called "temp"
-*.temp
-osm_common/temp
-osm_common/test/temp
-
-#distribution and package generation
-build
-dist
-*.egg-info
-*.egg-info
-.eggs
-
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..eb9750e
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,17 @@
+# Copyright Contributors to ETSI OSM
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+include:
+ - project: 'devops/cicd'
+ file: '/templates/osm-base.yml'
diff --git a/Dockerfile b/Dockerfile
index 43afa1c..3fad212 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,16 +10,27 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# This Dockerfile is intented for devops and deb package generation
+########################################################################################
+# This Dockerfile is intented for devops testing and deb package generation
#
-# Use Dockerfile.local for running osm/LCM in a docker container from source
+# To run stage 2 locally:
+#
+# docker build -t stage2 .
+# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2
+# devops-stages/stage-test.sh
+# devops-stages/stage-build.sh
+#
FROM ubuntu:18.04
-RUN apt-get update && apt-get -y install git make python python3 \
- libcurl4-gnutls-dev libgnutls28-dev tox python3-dev python3-pip \
- debhelper python-setuptools python-all python3-all apt-utils \
- python-magic && \
- DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb setuptools-version-command
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y install \
+ debhelper \
+ git \
+ python3 \
+ python3-all \
+ python3-dev \
+ python3-setuptools
+RUN python3 -m easy_install pip==21.0.1
+RUN pip3 install tox==3.22.0
diff --git a/MANIFEST.in b/MANIFEST.in
index cbc5d1e..e8fc479 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -17,5 +17,5 @@
##
include README.rst
-recursive-include osm_common *.py *.xml *.sh
+recursive-include osm_common *.py *.xml *.sh *.txt
recursive-include devops-stages *
\ No newline at end of file
diff --git a/debian/python3-osm-common.postinst b/debian/python3-osm-common.postinst
deleted file mode 100755
index fa43cbc..0000000
--- a/debian/python3-osm-common.postinst
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-##
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact with: OSM_TECH@list.etsi.org
-##
-
-echo "POST INSTALL OSM-COMMON"
-echo "Installing python dependencies via pip..."
-
-pip3 install dataclasses
-pip3 install --upgrade aiokafka
-pip3 install --upgrade pymongo
-pip3 install --upgrade pyyaml==5.*
-pip3 install pycrypto # added dependency python3-crypto seems not working
-
-#Creation of log folder
-mkdir -p /var/log/osm
-
diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh
index dfb7978..05b232e 100755
--- a/devops-stages/stage-build.sh
+++ b/devops-stages/stage-build.sh
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# moved to a Makefile in order to add post install. Needed for "pip3 install aiokafka",
-# that is not available with a package
+rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs .tox
-make clean package
+tox -e dist
diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh
index 2e7f488..e5a8e85 100755
--- a/devops-stages/stage-test.sh
+++ b/devops-stages/stage-test.sh
@@ -13,6 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-rm -f nosetests.xml
-tox # flake8 unittest pytest cover
-
+OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto)
+printf "$OUTPUT"
diff --git a/Makefile b/nose2.cfg
similarity index 62%
copy from Makefile
copy to nose2.cfg
index ed4ad8d..6823267 100644
--- a/Makefile
+++ b/nose2.cfg
@@ -1,4 +1,5 @@
-# Copyright 2018 Telefonica S.A.
+# -*- coding: utf-8 -*-
+# Copyright ETSI Contributors and Others.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,12 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-all: clean package
+[unittest]
+plugins = nose2.plugins.junitxml
-clean:
- rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs
-
-package:
- python3 setup.py --command-packages=stdeb.command sdist_dsc
- cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian
- cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us
+[junit-xml]
+always-on = True
+keep_restricted = False
+path = nosetests.xml
+test_fullname = False
diff --git a/Makefile b/requirements-dist.in
similarity index 62%
copy from Makefile
copy to requirements-dist.in
index ed4ad8d..11f0a2a 100644
--- a/Makefile
+++ b/requirements-dist.in
@@ -1,4 +1,4 @@
-# Copyright 2018 Telefonica S.A.
+# Copyright ETSI Contributors and Others.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,12 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-all: clean package
-
-clean:
- rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs
-
-package:
- python3 setup.py --command-packages=stdeb.command sdist_dsc
- cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian
- cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us
+stdeb
+setuptools-version-command
\ No newline at end of file
diff --git a/requirements-dist.txt b/requirements-dist.txt
new file mode 100644
index 0000000..7393626
--- /dev/null
+++ b/requirements-dist.txt
@@ -0,0 +1,23 @@
+setuptools-version-command==2.2
+ # via -r requirements-dist.in
+stdeb==0.10.0
+ # via -r requirements-dist.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/Makefile b/requirements-test.in
similarity index 62%
copy from Makefile
copy to requirements-test.in
index ed4ad8d..0378739 100644
--- a/Makefile
+++ b/requirements-test.in
@@ -1,4 +1,4 @@
-# Copyright 2018 Telefonica S.A.
+# Copyright ETSI Contributors and Others.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,12 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-all: clean package
-
-clean:
- rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs
-
-package:
- python3 setup.py --command-packages=stdeb.command sdist_dsc
- cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian
- cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us
+coverage
+nose2
+pytest
\ No newline at end of file
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..134c015
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,40 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+attrs==21.2.0
+ # via pytest
+coverage==5.5
+ # via
+ # -r requirements-test.in
+ # nose2
+iniconfig==1.1.1
+ # via pytest
+nose2==0.10.0
+ # via -r requirements-test.in
+packaging==20.9
+ # via pytest
+pluggy==0.13.1
+ # via pytest
+py==1.10.0
+ # via pytest
+pyparsing==2.4.7
+ # via packaging
+pytest==6.2.4
+ # via -r requirements-test.in
+six==1.16.0
+ # via nose2
+toml==0.10.2
+ # via pytest
diff --git a/Makefile b/requirements.in
similarity index 62%
rename from Makefile
rename to requirements.in
index ed4ad8d..1ce6a65 100644
--- a/Makefile
+++ b/requirements.in
@@ -1,4 +1,4 @@
-# Copyright 2018 Telefonica S.A.
+# Copyright ETSI Contributors and Others.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,12 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-all: clean package
-
-clean:
- rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs
-
-package:
- python3 setup.py --command-packages=stdeb.command sdist_dsc
- cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian
- cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us
+pymongo
+aiokafka
+pyyaml
+pycrypto
+dataclasses
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..2ca19f5
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,28 @@
+aiokafka==0.7.0
+ # via -r requirements.in
+dataclasses==0.6
+ # via -r requirements.in
+kafka-python==2.0.2
+ # via aiokafka
+pycrypto==2.6.1
+ # via -r requirements.in
+pymongo==3.11.3
+ # via -r requirements.in
+pyyaml==5.4.1
+ # via -r requirements.in
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/setup.py b/setup.py
index 157ffec..f53c839 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-# Copyright 2018 Telefonica S.A.
+# Copyright ETSI OSM Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -25,31 +25,21 @@
README = open(os.path.join(here, 'README.rst')).read()
setup(
- # python_requires='>=3.5',
name=_name,
description='OSM common utilities',
long_description=README,
version_command=('git describe --tags --long --dirty --match v*', 'pep440-git-full'),
- # version=VERSION,
- # python_requires='>3.5',
+
author='ETSI OSM',
- author_email='alfonso.tiernosepulveda@telefonica.com',
- maintainer='Alfonso Tierno',
- maintainer_email='alfonso.tiernosepulveda@telefonica.com',
+ author_email='osmsupport@etsi.com',
+ maintainer='ETSI OSM',
+ maintainer_email='osmsupport@etsi.com',
+
url='https://osm.etsi.org/gitweb/?p=osm/common.git;a=summary',
license='Apache 2.0',
setup_requires=['setuptools-version-command'],
packages=[_name],
include_package_data=True,
- # scripts=['nbi.py'],
- # dataclasses is required by aiokafka for Python <3.7.0
- install_requires=[
- 'pymongo',
- 'aiokafka',
- 'PyYAML',
- 'pycrypto',
- 'dataclasses',
- ],
)
diff --git a/stdeb.cfg b/stdeb.cfg
index 5bd8819..c6a1b3a 100644
--- a/stdeb.cfg
+++ b/stdeb.cfg
@@ -1,3 +1,17 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
[DEFAULT]
X-Python3-Version : >= 3.5
-Depends3 : python3-pip, python3-crypto
diff --git a/tox.ini b/tox.ini
index a7cf7b4..c3341ea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,4 +1,5 @@
-# Copyright 2018 Telefonica S.A.
+#######################################################################################
+# Copyright ETSI Contributors and Others.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,53 +13,103 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#######################################################################################
[tox]
-envlist = cover, flake8, unittest, pytest
+envlist = black, cover, flake8, pylint, safety
+
+[tox:jenkins]
+toxworkdir = /tmp/.tox
[testenv]
usedevelop = True
basepython = python3
+setenv = VIRTUAL_ENV={envdir}
+ PYTHONDONTWRITEBYTECODE = 1
+deps = -r{toxinidir}/requirements.txt
-[testenv:cover]
-basepython = python3
-deps =
- nose2
- nose2-cov
- coverage
- pytest
- pytest-asyncio
- pycrypto
+#######################################################################################
+[testenv:black]
+deps = black
+skip_install = true
commands =
- coverage erase
- nose2 -C --coverage osm_common --plugin nose2.plugins.junitxml -s osm_common/tests
- coverage report --omit='*tests*'
- coverage html -d ./cover --omit='*tests*'
- coverage xml -o coverage.xml --omit='*tests*'
+ - black --check --diff osm_common/
-[testenv:pytest]
-basepython = python3
-deps = pytest
- pytest-asyncio
- pycrypto
-commands = pytest osm_common
+#######################################################################################
+[testenv:cover]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-test.txt
+commands =
+ sh -c 'rm -f nosetests.xml'
+ coverage erase
+ nose2 -C --coverage osm_common -s osm_common/tests
+ coverage report --omit='*tests*'
+ coverage html -d ./cover --omit='*tests*'
+ coverage xml -o coverage.xml --omit=*tests*
+whitelist_externals = sh
+#######################################################################################
[testenv:flake8]
-basepython = python3
deps = flake8
commands =
- flake8 osm_common/ setup.py --max-line-length 120 --exclude .svn,CVS,.gz,.git,__pycache__,.tox,local,temp --ignore W291,W293,E226
+ flake8 osm_common/ setup.py
-[testenv:unittest]
-basepython = python3
-deps = pycrypto
- pytest
-commands = python3 -m unittest discover osm_common.tests
+#######################################################################################
+[testenv:pylint]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-test.txt
+ pylint
+commands =
+ pylint -E osm_common
-[testenv:build]
-basepython = python3
-deps = stdeb
- setuptools-version-command
-commands = python3 setup.py --command-packages=stdeb.command bdist_deb
+#######################################################################################
+[testenv:safety]
+setenv =
+ LC_ALL=C.UTF-8
+ LANG=C.UTF-8
+deps = {[testenv]deps}
+ safety
+commands =
+ - safety check --full-report
+#######################################################################################
+[testenv:pip-compile]
+deps = pip-tools==5.5.0
+commands =
+ - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\
+ out=`echo $file | sed "s/.in/.txt/"` ; \
+ head -16 tox.ini >> $out ;\
+ done'
+whitelist_externals = sh
+
+#######################################################################################
+[testenv:dist]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dist.txt
+
+# In the commands, we copy the requirements.txt to be presented as a source file (.py)
+# so it gets included in the .deb package for others to consume
+commands =
+ sh -c 'cp requirements.txt osm_common/requirements.txt'
+ python3 setup.py --command-packages=stdeb.command sdist_dsc
+ sh -c 'cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us'
+ sh -c 'rm osm_common/requirements.txt'
+whitelist_externals = sh
+
+#######################################################################################
+[flake8]
+ignore =
+ W291,
+ W293,
+ E123,
+ E125,
+ E226,
+ E241
+exclude =
+ .git,
+ __pycache__,
+ .tox,
+max-line-length = 120
+show-source = True
+builtins = _