Feature 10296 Pip Standardization
Creates standard template for tox.ini
Introduces 'safety' for listing upstream CVEs
Pins all versions of all upstream modules
Removes Makefile step and places all build logic into tox.ini
Change-Id: I841f19700b462aed919cf7bc55f1b0a38728ef3e
Signed-off-by: beierlm <mark.beierl@canonical.com>
diff --git a/.gitignore b/.gitignore
index 9007d45..de32f53 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,8 @@
#IDEs
.idea
.vscode
+.project
+.pydevproject
#Generated folders when building the deb package locally
.tox
@@ -30,8 +32,12 @@
deb_dist
pool
dist
+.coverage
+cover
+coverage.xml
#local stuff, e.g. files that end in ".local" or folders called "local"
*.local
local
venv
+
diff --git a/Dockerfile b/Dockerfile
index 64e8f27..e6b2f78 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,3 @@
-# Copyright 2020 ETSI
-#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -12,13 +10,30 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+########################################################################################
+# This Dockerfile is intented for devops testing and deb package generation
+#
+# To run stage 2 locally:
+#
+# docker build -t stage2 .
+# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2
+# devops-stages/stage-test.sh
+# devops-stages/stage-build.sh
+#
FROM ubuntu:18.04
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get -y install git tox make python3 python3-pip python-all && \
- DEBIAN_FRONTEND=noninteractive apt-get -y install python3-all debhelper python3-setuptools apt-utils libgl1-mesa-glx && \
- DEBIAN_FRONTEND=noninteractive pip3 install -U setuptools setuptools-version-command stdeb
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y install \
+ debhelper \
+ git \
+ python3 \
+ python3-all \
+ python3-dev \
+ python3-setuptools
+
+RUN python3 -m easy_install pip==21.0.1
+RUN pip3 install tox==3.22.0
ADD https://github.com/MiniZinc/MiniZincIDE/releases/download/2.4.2/MiniZincIDE-2.4.2-bundle-linux-x86_64.tgz /minizinc.tgz
@@ -31,7 +46,3 @@
ENV FZNEXEC "/entry_data/fzn-exec"
ENV PATH "/minizinc/bin:${PATH}"
-
-RUN mkdir /placement
-COPY ./osm_pla/test/pil_price_list.yaml /placement/.
-COPY ./osm_pla/test/vnf_price_list.yaml /placement/.
diff --git a/MANIFEST.in b/MANIFEST.in
index 60da55a..9e642ee 100755
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -14,7 +14,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
-include requirements.txt
+
include README.md
-recursive-include osm_pla *.py *.sh *.yaml *.j2
+recursive-include osm_pla *.py *.sh *.yaml *.j2 *.txt
recursive-include devops-stages *
diff --git a/Makefile b/Makefile
deleted file mode 100755
index 7e82f44..0000000
--- a/Makefile
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright 2020 ArctosLabs Scandinava AB
-# *************************************************************
-
-# This file is part of OSM Placement module
-# All Rights Reserved to ArctosLabs Scandinavia AB
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-##
-
-all: clean package
-
-clean:
- rm -rf dist deb_dist osm_pla-*.tar.gz osm_pla.egg-info .eggs
-
-package:
- python3 setup.py --command-packages=stdeb.command sdist_dsc
- cp debian/python3-osm-pla.postinst deb_dist/osm-pla*/debian
- cd deb_dist/osm-pla*/ && dpkg-buildpackage -rfakeroot -uc -us
diff --git a/debian/python3-osm-pla.postinst b/debian/python3-osm-pla.postinst
deleted file mode 100755
index bed12db..0000000
--- a/debian/python3-osm-pla.postinst
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-##
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact with: OSM_TECH@list.etsi.org
-##
-
-echo "POST INSTALL OSM-PLA"
-# Currently it is not needed pip3 installation
-# echo "Installing python dependencies via pip..."
-# pip3 install pip==9.0.3
-# pip3 install --user aiokafka
-
-#Creation of log folder
-mkdir -p /var/log/osm
-
-# systemctl enable osm-pla.service
diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh
index 9c2b16b..84e4819 100755
--- a/devops-stages/stage-build.sh
+++ b/devops-stages/stage-build.sh
@@ -12,4 +12,6 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-make
+rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs
+
+tox -e dist
\ No newline at end of file
diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh
index 69c6fcd..2d7b24e 100755
--- a/devops-stages/stage-test.sh
+++ b/devops-stages/stage-test.sh
@@ -11,4 +11,6 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-tox
+
+OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto)
+printf "$OUTPUT"
diff --git a/osm_pla/test/test_server.py b/osm_pla/test/test_server.py
index 9d7582f..fdde3eb 100644
--- a/osm_pla/test/test_server.py
+++ b/osm_pla/test/test_server.py
@@ -12,6 +12,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+# pylint: disable=E1120
+
import asyncio
import copy
import os
@@ -259,7 +262,7 @@
'mgmt-network': True,
'vim-network-name': 'lanretxe'
}],
- }
+ }
######################################################
diff --git a/requirements-dev.in b/requirements-dev.in
new file mode 100644
index 0000000..af8d95a
--- /dev/null
+++ b/requirements-dev.in
@@ -0,0 +1,16 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..485e45e
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,30 @@
+aiokafka==0.7.0
+ # via osm-common
+dataclasses==0.6
+ # via osm-common
+kafka-python==2.0.2
+ # via aiokafka
+git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common
+ # via -r requirements-dev.in
+pycrypto==2.6.1
+ # via osm-common
+pymongo==3.11.3
+ # via osm-common
+pyyaml==5.4.1
+ # via osm-common
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/requirements-dist.in b/requirements-dist.in
new file mode 100644
index 0000000..11f0a2a
--- /dev/null
+++ b/requirements-dist.in
@@ -0,0 +1,17 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stdeb
+setuptools-version-command
\ No newline at end of file
diff --git a/requirements-dist.txt b/requirements-dist.txt
new file mode 100644
index 0000000..7393626
--- /dev/null
+++ b/requirements-dist.txt
@@ -0,0 +1,23 @@
+setuptools-version-command==2.2
+ # via -r requirements-dist.in
+stdeb==0.10.0
+ # via -r requirements-dist.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/test-requirements.txt b/requirements-test.in
similarity index 97%
rename from test-requirements.txt
rename to requirements-test.in
index 429714f..9a0542d 100755
--- a/test-requirements.txt
+++ b/requirements-test.in
@@ -15,4 +15,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
-coverage
\ No newline at end of file
+
+coverage
+mock
+nose2
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..d0c984f
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,26 @@
+coverage==5.5
+ # via
+ # -r requirements-test.in
+ # nose2
+mock==4.0.3
+ # via -r requirements-test.in
+nose2==0.10.0
+ # via -r requirements-test.in
+six==1.15.0
+ # via nose2
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/test-requirements.txt b/requirements.in
similarity index 95%
copy from test-requirements.txt
copy to requirements.in
index 429714f..82788ed 100755
--- a/test-requirements.txt
+++ b/requirements.in
@@ -15,4 +15,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
-coverage
\ No newline at end of file
+pyyaml
+pymzn==0.18.*
+jinja2
diff --git a/requirements.txt b/requirements.txt
old mode 100755
new mode 100644
index 960f499..6f8103b
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,21 +1,24 @@
-# Copyright 2020 ArctosLabs Scandinavia AB
-# *************************************************************
-
-# This file is part of OSM Placement module
-# All Rights Reserved to ArctosLabs Scandinavia AB
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
+jinja2==2.11.3
+ # via -r requirements.in
+markupsafe==1.1.1
+ # via jinja2
+pymzn==0.18.3
+ # via -r requirements.in
+pyyaml==5.4.1
+ # via -r requirements.in
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-pyyaml==5.1.2
-pymzn==0.18.*
-jinja2==2.10.3
-git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common
\ No newline at end of file
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/setup.py b/setup.py
index 2c7cd04..bb062d7 100755
--- a/setup.py
+++ b/setup.py
@@ -23,10 +23,10 @@
_name = 'osm_pla'
_version_command = ('git describe --match v* --tags --long --dirty', 'pep440-git-full')
_description = 'OSM Placement Module'
-_author = "Lars Goran Magnusson"
-_author_email = 'lars-goran.magnusson@arctoslabs.com'
-_maintainer = 'Lars Goran Magnusson'
-_maintainer_email = 'lars-goran.magnusson@arctoslabs.com'
+_author = "OSM Support"
+_author_email = 'osmsupport@etsi.org'
+_maintainer = 'OSM Support'
+_maintainer_email = 'osmsupport@etsi.org'
_license = 'Apache 2.0'
_url = 'https://osm.etsi.org/gitweb?p=osm/PLA.git;a=tree'
@@ -44,15 +44,6 @@
license=_license,
packages=[_name],
package_dir={_name: _name},
- install_requires=[
- 'osm-common',
- 'jinja2==2.10.3',
- 'pymzn==0.18.3',
- 'pyyaml==5.1.2'
- ],
- dependency_links=[
- 'git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common',
- ],
include_package_data=True,
entry_points={
"console_scripts": [
diff --git a/stdeb.cfg b/stdeb.cfg
index 05381db..017d71c 100644
--- a/stdeb.cfg
+++ b/stdeb.cfg
@@ -12,6 +12,6 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
[DEFAULT]
X-Python3-Version : >= 3.5
-Depends3: python3-osm-common, python3-yaml, python3-jinja2, python3-pip
diff --git a/tox.ini b/tox.ini
index 68c6211..d244554 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,50 +1,124 @@
-##
-# Copyright 2020 ArctosLabs Scandinavia AB
+#######################################################################################
+# Copyright ETSI Contributors and Others.
#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-##
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
[tox]
-envlist = py3
+envlist = black, cover, flake8, pylint, safety
+
+[tox:jenkins]
+toxworkdir = /tmp/.tox
[testenv]
+usedevelop = True
basepython = python3
-install_command = python3 -m pip install -U {opts} {packages}
-deps = -rrequirements.txt
- -rtest-requirements.txt
-commands = python3 -m unittest discover -v
+setenv = VIRTUAL_ENV={envdir}
+ PYTHONDONTWRITEBYTECODE = 1
+deps = -r{toxinidir}/requirements.txt
-[testenv:coverage]
-basepython = python3
-deps = -rrequirements.txt
- -rtest-requirements.txt
-commands = coverage run -m unittest discover
- coverage report --omit='*site-packages*','*test*','*__init__*'
- coverage html -d ./.tox/coverage/HTMLreport --omit='*site-packages*','*test*','*__init__*'
- coverage xml -o ./.tox/coverage/XMLreport/coverage.xml --omit='*site-packages*','*test*','*__init__*'
- coverage erase
+#######################################################################################
+[testenv:black]
+deps = black
+skip_install = true
+commands =
+ - black --check --diff osm_pla/
+
+#######################################################################################
+[testenv:cover]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dev.txt
+ -r{toxinidir}/requirements-test.txt
+commands =
+ sh -c 'rm -f nosetests.xml'
+ coverage erase
+ nose2 -C --coverage osm_pla
+ coverage report --omit='*tests*'
+ coverage html -d ./cover --omit='*tests*'
+ coverage xml -o coverage.xml --omit=*tests*
+whitelist_externals = sh
+
+
+#######################################################################################
[testenv:flake8]
-basepython = python3
deps = flake8
- -rrequirements.txt
-commands = flake8 {toxinidir}/osm_pla/ {toxinidir}/setup.py \
- --max-line-length 120 \
- --exclude test_mznmodels.py,.svn,CVS,.gz,.git,__pycache__,.tox,local,temp
+commands =
+ flake8 osm_pla/ setup.py
-[testenv:build]
-basepython = python3
-# changedir ={toxinidir}
-deps = stdeb
- setuptools-version-command
- -rrequirements.txt
-commands = python3 setup.py --command-packages=stdeb.command bdist_deb
+
+#######################################################################################
+[testenv:pylint]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dev.txt
+ -r{toxinidir}/requirements-test.txt
+ pylint
+commands =
+ pylint -E osm_pla
+
+
+#######################################################################################
+[testenv:safety]
+setenv =
+ LC_ALL=C.UTF-8
+ LANG=C.UTF-8
+deps = {[testenv]deps}
+ safety
+commands =
+ - safety check --full-report
+
+
+#######################################################################################
+[testenv:pip-compile]
+deps = pip-tools==5.5.0
+commands =
+ - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\
+ out=`echo $file | sed "s/.in/.txt/"` ; \
+ head -16 tox.ini >> $out ;\
+ done'
+whitelist_externals = sh
+
+
+#######################################################################################
+[testenv:dist]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dist.txt
+
+# In the commands, we copy the requirements.txt to be presented as a source file (.py)
+# so it gets included in the .deb package for others to consume
+commands =
+ sh -c 'cp requirements.txt osm_pla/requirements.txt'
+ python3 setup.py --command-packages=stdeb.command sdist_dsc
+ sh -c 'cd deb_dist/osm-pla*/ && dpkg-buildpackage -rfakeroot -uc -us'
+ sh -c 'rm osm_pla/requirements.txt'
+whitelist_externals = sh
+
+#######################################################################################
+[flake8]
+ignore =
+ W291,
+ W293,
+ W503,
+ E123,
+ E125,
+ E226,
+ E241
+exclude =
+ .git,
+ __pycache__,
+ .tox,
+ test_mznmodels.py
+max-line-length = 120
+show-source = True
+builtins = _