From 5b3f4b6983b2957df33e5995ce32f3351a089290 Mon Sep 17 00:00:00 2001 From: beierlm Date: Wed, 17 Feb 2021 07:22:20 -0500 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Removes dependencies from Makefile and Dockerfile and places them into requirements.txt Change-Id: If2c52b101bad455534096567b4d68c2baba05d22 Signed-off-by: beierlm --- Dockerfile | 55 +++++++++++------- MANIFEST.in | 1 + Makefile | 16 ++--- devops-stages/stage-build.sh | 21 ++++++- devops-stages/stage-test.sh | 5 +- requirements-dist.in | 17 ++++++ requirements-dist.txt | 23 ++++++++ requirements-test.in | 16 +++++ requirements-test.txt | 24 ++++++++ requirements.in | 18 ++++++ requirements.txt | 36 ++++++++++++ setup.py | 5 +- stdeb.cfg | 6 +- tox.ini | 110 ++++++++++++++++++++++++++++++----- 14 files changed, 292 insertions(+), 61 deletions(-) create mode 100644 requirements-dist.in create mode 100644 requirements-dist.txt create mode 100644 requirements-test.in create mode 100644 requirements-test.txt create mode 100644 requirements.in create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index a5c912e..c811fb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,40 @@ -# Copyright 2017 Telefonica Investigacion y Desarrollo, S.A.U. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# All Rights Reserved. +# http://www.apache.org/licenses/LICENSE-2.0 # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation # -# http://www.apache.org/licenses/LICENSE-2.0 +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. FROM ubuntu:18.04 -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install git make wget python \ - python3 python3-all python3-pip python-pip \ - debhelper tox python-setuptools \ - python3-setuptools build-essential dh-make \ - openjdk-8-jdk maven && \ - update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64 && \ - DEBIAN_FRONTEND=noninteractive pip3 install pip==9.0.3 && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U pyang pyangbind && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb && \ - DEBIAN_FRONTEND=noninteractive pip2 install -U stdeb +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 + +RUN DEBIAN_FRONTEND=noninteractive apt -y install wget build-essential dh-make \ + openjdk-8-jdk maven +RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64 diff --git a/MANIFEST.in b/MANIFEST.in index b804618..a4fe252 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +recursive-include osm_im *.py *.txt recursive-include models *.yang \ No newline at end of file diff --git a/Makefile b/Makefile index be0faa3..703f73c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ # Copyright 2017 Sandvine # Copyright 2017-2018 Telefonica # All Rights Reserved. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -105,17 +105,11 @@ yang2swagger: mvn -f yang2swagger/pom.xml clean install package: - tox -e build ./build-docs.sh deps: - $(Q)sudo apt-get -y install git make wget python python-pip debhelper dh-make tox python3 python3-pip maven - $(Q)sudo -H python3 -m pip install -U pip - $(Q)sudo -H python2 -m pip install -U pip - $(Q)sudo -H python3 -m pip install -U pyang pyangbind stdeb - $(Q)sudo -H python2 -m pip install -U pyang pyangbind stdeb - $(Q)mkdir -p ~/.m2 - $(Q)cp -n ~/.m2/settings.xml ~/.m2/settings.xml.orig ; wget -q -O - https://raw.githubusercontent.com/opendaylight/odlparent/master/settings.xml > ~/.m2/settings.xml + $(Q)mkdir -p ${HOME}/.m2 + $(Q)cp -n ${HOME}/.m2/settings.xml ${HOME}/.m2/settings.xml.orig ; wget -q -O - https://raw.githubusercontent.com/opendaylight/odlparent/master/settings.xml > ${HOME}/.m2/settings.xml sol006_deps: $(Q)git clone --single-branch --branch v2.6.1 https://forge.etsi.org/rep/nfv/SOL006.git sol006_model diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 2c0c17f..e8b397b 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -1,3 +1,18 @@ -#!/bin/sh -make clean -make +#!/bin/sh -x + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs + +tox -e dist diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 36bb8d3..067d896 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -16,6 +16,5 @@ # License for the specific language governing permissions and limitations # under the License. -make clean -make models -tox +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100644 index 0000000..43bf361 --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,16 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +coverage +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..decfef2 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,24 @@ +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +nose2==0.10.0 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..24f4c58 --- /dev/null +++ b/requirements.in @@ -0,0 +1,18 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pyang +pyangbind +pyyaml \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..002200e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,36 @@ +bitarray==1.8.1 + # via pyangbind +enum34==1.1.10 + # via pyangbind +lxml==4.6.3 + # via + # pyang + # pyangbind +pyang==2.4.0 + # via + # -r requirements.in + # pyangbind +pyangbind==0.8.1 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +regex==2021.3.17 + # via pyangbind +six==1.15.0 + # via pyangbind +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index dfa5cee..71c7eae 100644 --- a/setup.py +++ b/setup.py @@ -48,12 +48,11 @@ setup( description='OSM Information Model', long_description=open('README.rst').read(), version_command=('git describe --tags --long --dirty --match v*', 'pep440-git-full'), - author='Mike Marchetti', - author_email='mmarchetti@sandvine.com', + author='OSM Support', + author_email='osmsupport@etsi.org', packages=find_packages(), include_package_data=True, setup_requires=['setuptools-version-command'], - install_requires=['pyang', 'pyangbind'], test_suite='nose.collector', url='https://osm.etsi.org/gitweb/?p=osm/IM.git;a=summary', license='Apache 2.0', diff --git a/stdeb.cfg b/stdeb.cfg index ec4c256..6fef760 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -13,8 +13,4 @@ # limitations under the License. [DEFAULT] -Suite: xenial -Build-Depends: dh-python -Maintainer: Gerardo Garcia -Depends: osm-imdocs -Depends3: osm-imdocs +X-Python3-Version : >= 3.5 diff --git a/tox.ini b/tox.ini index d8d9e3b..0810474 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,6 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,26 +13,105 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +####################################################################################### [tox] -envlist = py3 -toxworkdir={homedir}/.tox +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] -deps=nose - mock - pyyaml -commands=nosetests +usedevelop = True +basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +passenv = HOME +deps = -r{toxinidir}/requirements.txt + + +####################################################################################### +[testenv:black] +commands = python3 -c 'print("Skipping due to generated code")' + +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'make clean' + sh -c 'make models Q=""' + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osm_im -s tests + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + + +####################################################################################### [testenv:flake8] -basepython = python3 -deps = flake8 +commands = python3 -c 'print("Skipping due to generated code")' + + +####################################################################################### +[testenv:pylint] +commands = python3 -c 'print("Skipping due to generated code")' + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety commands = - flake8 setup.py + - safety check --full-report -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command -commands = python3 setup.py --command-packages=stdeb.command bdist_deb +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'make clean' + sh -c 'make Q=""' + sh -c 'cp requirements.txt osm_im/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-im*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osm_im/requirements.txt' +whitelist_externals = sh + +####################################################################################### +[flake8] +ignore = + W291, + W293, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 +show-source = True +builtins = _ -- 2.17.1