Feature 10296 Pip Standardization

Creates standard template for tox.ini
Introduces 'safety' for listing upstream CVEs
Pins all versions of all upstream modules
Removes dependencies from Makefile and Dockerfile
and places them into requirements.txt

Change-Id: If2c52b101bad455534096567b4d68c2baba05d22
Signed-off-by: beierlm <mark.beierl@canonical.com>

diff --git a/Dockerfile b/Dockerfile
index a5c912e..c811fb7 100644 (file)
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,29 +1,40 @@
-# Copyright 2017 Telefonica Investigacion y Desarrollo, S.A.U.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# All Rights Reserved.
+#    http://www.apache.org/licenses/LICENSE-2.0
 #
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################################
+# This Dockerfile is intented for devops testing and deb package generation
 #
-#         http://www.apache.org/licenses/LICENSE-2.0
+# To run stage 2 locally:
+#
+#   docker build -t stage2 .
+#   docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2
+#   devops-stages/stage-test.sh
+#   devops-stages/stage-build.sh
 #
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
 
 FROM ubuntu:18.04
 
-RUN  apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get -y install git make wget python \
-                                            python3 python3-all python3-pip python-pip \
-                                            debhelper tox python-setuptools \
-                                            python3-setuptools build-essential dh-make \
-                                            openjdk-8-jdk maven && \
-  update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64 && \
-  DEBIAN_FRONTEND=noninteractive pip3 install pip==9.0.3 && \
-  DEBIAN_FRONTEND=noninteractive pip3 install -U pyang pyangbind && \
-  DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb && \
-  DEBIAN_FRONTEND=noninteractive pip2 install -U stdeb
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -y install \
+        debhelper \
+        git \
+        python3 \
+        python3-all \
+        python3-dev \
+        python3-setuptools
+
+RUN python3 -m easy_install pip==21.0.1
+RUN pip3 install tox==3.22.0
+
+RUN DEBIAN_FRONTEND=noninteractive apt -y install wget build-essential dh-make \
+                                            openjdk-8-jdk maven
+RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64

diff --git a/MANIFEST.in b/MANIFEST.in
index b804618..a4fe252 100644 (file)
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -12,4 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+recursive-include osm_im *.py *.txt
 recursive-include models *.yang
\ No newline at end of file

diff --git a/Makefile b/Makefile
index be0faa3..703f73c 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,13 @@
 # Copyright 2017 Sandvine
 # Copyright 2017-2018 Telefonica
 # All Rights Reserved.
-# 
+#
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
-# 
+#
 #         http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 #    Unless required by applicable law or agreed to in writing, software
 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
@@ -105,17 +105,11 @@ yang2swagger:
        mvn -f yang2swagger/pom.xml clean install
 
 package:
-       tox -e build
        ./build-docs.sh
 
 deps:
-       $(Q)sudo apt-get -y install git make wget python python-pip debhelper dh-make tox python3 python3-pip maven
-       $(Q)sudo -H python3 -m pip install -U pip
-       $(Q)sudo -H python2 -m pip install -U pip
-       $(Q)sudo -H python3 -m pip install -U pyang pyangbind stdeb
-       $(Q)sudo -H python2 -m pip install -U pyang pyangbind stdeb
-       $(Q)mkdir -p ~/.m2
-       $(Q)cp -n ~/.m2/settings.xml ~/.m2/settings.xml.orig ; wget -q -O - https://raw.githubusercontent.com/opendaylight/odlparent/master/settings.xml > ~/.m2/settings.xml
+       $(Q)mkdir -p ${HOME}/.m2
+       $(Q)cp -n ${HOME}/.m2/settings.xml ${HOME}/.m2/settings.xml.orig ; wget -q -O - https://raw.githubusercontent.com/opendaylight/odlparent/master/settings.xml > ${HOME}/.m2/settings.xml
 
 sol006_deps:
        $(Q)git clone --single-branch --branch v2.6.1 https://forge.etsi.org/rep/nfv/SOL006.git sol006_model

diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh
index 2c0c17f..e8b397b 100755 (executable)
--- a/devops-stages/stage-build.sh
+++ b/devops-stages/stage-build.sh
@@ -1,3 +1,18 @@
-#!/bin/sh
-make clean
-make
+#!/bin/sh -x
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs
+
+tox -e dist

diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh
index 36bb8d3..067d896 100755 (executable)
--- a/devops-stages/stage-test.sh
+++ b/devops-stages/stage-test.sh
@@ -16,6 +16,5 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-make clean
-make models
-tox
+OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto)
+printf "$OUTPUT"

diff --git a/requirements-dist.in b/requirements-dist.in
new file mode 100644 (file)
index 0000000..11f0a2a
--- /dev/null
+++ b/requirements-dist.in
@@ -0,0 +1,17 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stdeb
+setuptools-version-command
\ No newline at end of file

diff --git a/requirements-dist.txt b/requirements-dist.txt
new file mode 100644 (file)
index 0000000..7393626
--- /dev/null
+++ b/requirements-dist.txt
@@ -0,0 +1,23 @@
+setuptools-version-command==2.2
+    # via -r requirements-dist.in
+stdeb==0.10.0
+    # via -r requirements-dist.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################

diff --git a/requirements-test.in b/requirements-test.in
new file mode 100644 (file)
index 0000000..43bf361
--- /dev/null
+++ b/requirements-test.in
@@ -0,0 +1,16 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+coverage
+nose2

diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644 (file)
index 0000000..decfef2
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,24 @@
+coverage==5.5
+    # via
+    #   -r requirements-test.in
+    #   nose2
+nose2==0.10.0
+    # via -r requirements-test.in
+six==1.15.0
+    # via nose2
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################

diff --git a/requirements.in b/requirements.in
new file mode 100644 (file)
index 0000000..24f4c58
--- /dev/null
+++ b/requirements.in
@@ -0,0 +1,18 @@
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pyang
+pyangbind
+pyyaml
\ No newline at end of file

diff --git a/requirements.txt b/requirements.txt
new file mode 100644 (file)
index 0000000..002200e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,36 @@
+bitarray==1.8.1
+    # via pyangbind
+enum34==1.1.10
+    # via pyangbind
+lxml==4.6.3
+    # via
+    #   pyang
+    #   pyangbind
+pyang==2.4.0
+    # via
+    #   -r requirements.in
+    #   pyangbind
+pyangbind==0.8.1
+    # via -r requirements.in
+pyyaml==5.4.1
+    # via -r requirements.in
+regex==2021.3.17
+    # via pyangbind
+six==1.15.0
+    # via pyangbind
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################

diff --git a/setup.py b/setup.py
index dfa5cee..71c7eae 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -48,12 +48,11 @@ setup(
     description='OSM Information Model',
     long_description=open('README.rst').read(),
     version_command=('git describe --tags --long --dirty --match v*', 'pep440-git-full'),
-    author='Mike Marchetti',
-    author_email='mmarchetti@sandvine.com',
+    author='OSM Support',
+    author_email='osmsupport@etsi.org',
     packages=find_packages(),
     include_package_data=True,
     setup_requires=['setuptools-version-command'],
-    install_requires=['pyang', 'pyangbind'],
     test_suite='nose.collector',
     url='https://osm.etsi.org/gitweb/?p=osm/IM.git;a=summary',
     license='Apache 2.0',

diff --git a/stdeb.cfg b/stdeb.cfg
index ec4c256..6fef760 100644 (file)
--- a/stdeb.cfg
+++ b/stdeb.cfg
@@ -13,8 +13,4 @@
 # limitations under the License.
 
 [DEFAULT]
-Suite: xenial
-Build-Depends: dh-python
-Maintainer: Gerardo Garcia <gerardo.garciadeblas@telefonica.com>
-Depends: osm-imdocs
-Depends3: osm-imdocs
+X-Python3-Version : >= 3.5

diff --git a/tox.ini b/tox.ini
index d8d9e3b..0810474 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -1,3 +1,6 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,26 +13,105 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#######################################################################################
 
 [tox]
-envlist = py3
-toxworkdir={homedir}/.tox
+envlist = black, cover, flake8, pylint, safety
+
+[tox:jenkins]
+toxworkdir = /tmp/.tox
 
 [testenv]
-deps=nose
-     mock
-     pyyaml
-commands=nosetests
+usedevelop = True
+basepython = python3
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONDONTWRITEBYTECODE = 1
+passenv = HOME
+deps =  -r{toxinidir}/requirements.txt
+
+
+#######################################################################################
+[testenv:black]
+commands = python3 -c 'print("Skipping due to generated code")'
+
 
+#######################################################################################
+[testenv:cover]
+deps =  {[testenv]deps}
+        -r{toxinidir}/requirements-test.txt
+commands =
+        sh -c 'make clean'
+        sh -c 'make models Q=""'
+        sh -c 'rm -f nosetests.xml'
+        coverage erase
+        nose2 -C --coverage osm_im -s tests
+        coverage report --omit='*tests*'
+        coverage html -d ./cover --omit='*tests*'
+        coverage xml -o coverage.xml --omit=*tests*
+whitelist_externals = sh
+
+
+#######################################################################################
 [testenv:flake8]
-basepython = python3
-deps = flake8
+commands = python3 -c 'print("Skipping due to generated code")'
+
+
+#######################################################################################
+[testenv:pylint]
+commands = python3 -c 'print("Skipping due to generated code")'
+
+
+#######################################################################################
+[testenv:safety]
+setenv =
+        LC_ALL=C.UTF-8
+        LANG=C.UTF-8
+deps =  {[testenv]deps}
+        safety
 commands =
-    flake8 setup.py
+        - safety check --full-report
 
-[testenv:build]
-basepython = python3
-deps = stdeb
-       setuptools-version-command
-commands = python3 setup.py --command-packages=stdeb.command bdist_deb
 
+#######################################################################################
+[testenv:pip-compile]
+deps =  pip-tools==5.5.0
+commands =
+        - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\
+        out=`echo $file | sed "s/.in/.txt/"` ; \
+        head -16 tox.ini >> $out ;\
+        done'
+whitelist_externals = sh
+
+
+#######################################################################################
+[testenv:dist]
+deps =  {[testenv]deps}
+        -r{toxinidir}/requirements-dist.txt
+
+# In the commands, we copy the requirements.txt to be presented as a source file (.py)
+# so it gets included in the .deb package for others to consume
+commands =
+        sh -c 'make clean'
+        sh -c 'make Q=""'
+        sh -c 'cp requirements.txt osm_im/requirements.txt'
+        python3 setup.py --command-packages=stdeb.command sdist_dsc
+        sh -c 'cd deb_dist/osm-im*/ && dpkg-buildpackage -rfakeroot -uc -us'
+        sh -c 'rm osm_im/requirements.txt'
+whitelist_externals = sh
+
+#######################################################################################
+[flake8]
+ignore =
+        W291,
+        W293,
+        E123,
+        E125,
+        E226,
+        E241
+exclude =
+        .git,
+        __pycache__,
+        .tox,
+max-line-length = 120
+show-source = True
+builtins = _