From a0911d313aed81603a71ec70a7fe6835f734961f Mon Sep 17 00:00:00 2001 From: beierlm Date: Mon, 22 Feb 2021 14:30:35 -0500 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Change-Id: I156239b97cd550c5349eb40197d85ce1a8dfde6c Signed-off-by: beierlm --- Dockerfile | 39 ++-- MANIFEST.in | 2 +- devops-stages/stage-build.sh | 6 +- devops-stages/stage-test.sh | 5 +- requirements-dev.in | 14 ++ requirements-dev.txt | 50 +++++ requirements-dist.in | 17 ++ Makefile => requirements-dist.txt | 19 +- test-requirements.txt => requirements-test.in | 10 +- requirements-test.txt | 60 ++++++ requirements.in | 19 ++ requirements.txt | 178 ++++++++++++++++-- setup.py | 20 +- stdeb.cfg | 2 - tox.ini | 124 +++++++++--- 15 files changed, 456 insertions(+), 109 deletions(-) create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements-dist.in rename Makefile => requirements-dist.txt (55%) rename test-requirements.txt => requirements-test.in (91%) create mode 100644 requirements-test.txt create mode 100644 requirements.in diff --git a/Dockerfile b/Dockerfile index 24efbad..ebfab05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -# Copyright 2018 Telefonica S.A. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,25 +10,30 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -# This Dockerfile is intented for devops and deb package generation +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Use Dockerfile.local for running osm/NBI in a docker container from source -# Use Dockerfile.fromdeb for running osm/NBI in a docker container from last stable package - FROM ubuntu:18.04 -RUN apt-get update && apt-get -y install wget git make python python3 \ - libcurl4-gnutls-dev libgnutls28-dev tox python3-dev python3-pip \ - debhelper python-setuptools python-all python3-all apt-utils \ - python-magic && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb setuptools-version-command +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 -# Uncomment this block to generate automatically a debian package and show info -# # Set the working directory to /app -# WORKDIR /app -# # Copy the current directory contents into the container at /app -# ADD . /app -# CMD /app/devops-stages/stage-build.sh && find -name "*.deb" -exec dpkg -I {} ";" +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget diff --git a/MANIFEST.in b/MANIFEST.in index 2e06a57..8615938 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,7 +17,7 @@ ## include README.rst -recursive-include osm_nbi *.py *.sh *.cfg *.yml +recursive-include osm_nbi *.py *.sh *.cfg *.yml *.txt recursive-include osm_nbi/html_public * recursive-include osm_nbi/http * recursive-include devops-stages * diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 0ce2bb2..9e78a73 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -make clean -# tox -e build -make package +rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs + +tox -e dist \ No newline at end of file diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 87802ea..e5a8e85 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -13,6 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -rm -f nosetests.xml -tox # flake8 unittest - +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..dcaa6bf --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common +git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..fa3825a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,50 @@ +aiokafka==0.7.0 + # via osm-common +bitarray==1.8.1 + # via pyangbind +dataclasses==0.6 + # via osm-common +enum34==1.1.10 + # via pyangbind +kafka-python==2.0.2 + # via aiokafka +lxml==4.6.3 + # via + # pyang + # pyangbind +git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common + # via -r requirements-dev.in +git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im + # via -r requirements-dev.in +pyang==2.4.0 + # via + # osm-im + # pyangbind +pyangbind==0.8.1 + # via osm-im +pycrypto==2.6.1 + # via osm-common +pymongo==3.11.3 + # via osm-common +pyyaml==5.4.1 + # via osm-common +regex==2021.3.17 + # via pyangbind +six==1.15.0 + # via pyangbind +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/Makefile b/requirements-dist.txt similarity index 55% rename from Makefile rename to requirements-dist.txt index 528cca1..c8f13c9 100644 --- a/Makefile +++ b/requirements-dist.txt @@ -1,3 +1,10 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools # Copyright 2018 Telefonica S.A. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,16 +20,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: clean package - -clean: - rm -rf dist deb_dist osm_nbi-*.tar.gz osm_nbi.egg-info .eggs .temp-tox - -package: - python3 setup.py --command-packages=stdeb.command sdist_dsc - cp debian/python3-osm-nbi.postinst deb_dist/osm-nbi*/debian - # cd deb_dist/osm-nbi*/debian && echo "osm-common python3-osm-common" > py3dist-overrides - # cd deb_dist/osm-nbi*/debian && echo "pip3 python3-pip" >> py3dist-overrides - cd deb_dist/osm-nbi*/ && dpkg-buildpackage -rfakeroot -uc -us - diff --git a/test-requirements.txt b/requirements-test.in similarity index 91% rename from test-requirements.txt rename to requirements-test.in index 560046d..4564164 100644 --- a/test-requirements.txt +++ b/requirements-test.in @@ -11,10 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# flake8<3.0 -# mock -# pyangbind -pyang +aiohttp>=2.3.10,<=3.6.2 aioresponses asynctest - +coverage +nose2 +requests==2.25.1 +pyang diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..b4cf255 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,60 @@ +aiohttp==3.6.2 + # via + # -r requirements-test.in + # aioresponses +aioresponses==0.7.2 + # via -r requirements-test.in +async-timeout==3.0.1 + # via aiohttp +asynctest==0.13.0 + # via -r requirements-test.in +attrs==20.3.0 + # via aiohttp +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via + # aiohttp + # requests +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +idna==2.10 + # via + # requests + # yarl +lxml==4.6.3 + # via pyang +multidict==4.7.6 + # via + # aiohttp + # yarl +nose2==0.10.0 + # via -r requirements-test.in +pyang==2.4.0 + # via -r requirements-test.in +requests==2.25.1 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +urllib3==1.26.4 + # via requests +yarl==1.6.3 + # via aiohttp +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..8098683 --- /dev/null +++ b/requirements.in @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +aiohttp>=2.3.10,<=3.6.2 +CherryPy>=18.1.2 +jsonschema>=3.2.0 +python-keystoneclient +pyyaml +requests +tacacs_plus diff --git a/requirements.txt b/requirements.txt index b51ddca..1861ebd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,163 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 +aiohttp==3.6.2 + # via -r requirements.in +async-timeout==3.0.1 + # via aiohttp +attrs==20.3.0 + # via + # aiohttp + # jsonschema +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via + # aiohttp + # requests +cheroot==8.5.2 + # via cherrypy +cherrypy==18.6.0 + # via -r requirements.in +debtcollector==2.2.0 + # via + # oslo.config + # oslo.utils + # python-keystoneclient +idna==2.10 + # via + # requests + # yarl +iso8601==0.1.14 + # via + # keystoneauth1 + # oslo.utils +jaraco.classes==3.2.1 + # via jaraco.collections +jaraco.collections==3.2.0 + # via cherrypy +jaraco.functools==3.2.1 + # via + # cheroot + # jaraco.text + # tempora +jaraco.text==3.5.0 + # via jaraco.collections +jsonschema==3.2.0 + # via -r requirements.in +keystoneauth1==4.3.1 + # via python-keystoneclient +more-itertools==8.7.0 + # via + # cheroot + # cherrypy + # jaraco.classes + # jaraco.functools +msgpack==1.0.2 + # via oslo.serialization +multidict==4.7.6 + # via + # aiohttp + # yarl +netaddr==0.8.0 + # via + # oslo.config + # oslo.utils +netifaces==0.10.9 + # via oslo.utils +os-service-types==1.7.0 + # via keystoneauth1 +oslo.config==8.5.0 + # via python-keystoneclient +oslo.i18n==5.0.1 + # via + # oslo.config + # oslo.utils + # python-keystoneclient +oslo.serialization==4.1.0 + # via python-keystoneclient +oslo.utils==4.8.0 + # via + # oslo.serialization + # python-keystoneclient +packaging==20.9 + # via oslo.utils +pbr==5.5.1 + # via + # debtcollector + # keystoneauth1 + # os-service-types + # oslo.i18n + # oslo.serialization + # oslo.utils + # python-keystoneclient + # stevedore +portend==2.7.1 + # via cherrypy +pyparsing==2.4.7 + # via + # oslo.utils + # packaging +pyrsistent==0.17.3 + # via jsonschema +python-keystoneclient==4.2.0 + # via -r requirements.in +pytz==2021.1 + # via + # oslo.serialization + # oslo.utils + # tempora +pyyaml==5.4.1 + # via + # -r requirements.in + # oslo.config +requests==2.25.1 + # via + # -r requirements.in + # keystoneauth1 + # oslo.config + # python-keystoneclient +rfc3986==1.4.0 + # via oslo.config +six==1.15.0 + # via + # cheroot + # debtcollector + # jsonschema + # keystoneauth1 + # oslo.i18n + # python-keystoneclient + # tacacs-plus +stevedore==3.3.0 + # via + # keystoneauth1 + # oslo.config + # python-keystoneclient +tacacs-plus==2.6 + # via -r requirements.in +tempora==4.0.1 + # via portend +urllib3==1.26.4 + # via requests +wrapt==1.12.1 + # via debtcollector +yarl==1.6.3 + # via aiohttp +zc.lockfile==2.0 + # via cherrypy +# The following packages are considered to be unsafe in a requirements file: +# setuptools +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -CherryPy>=18.1.2 -jsonschema>=3.2.0 -PyYAML -python-keystoneclient -requests -git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common -git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im -aiohttp>=2.3.10,<=3.6.2 -tacacs_plus diff --git a/setup.py b/setup.py index 309dda1..ad987ac 100644 --- a/setup.py +++ b/setup.py @@ -35,27 +35,13 @@ setup( # version=VERSION, # python_requires='>3.5.0', author='ETSI OSM', - author_email='alfonso.tiernosepulveda@telefonica.com', - maintainer='Alfonso Tierno', - maintainer_email='alfonso.tiernosepulveda@telefonica.com', + author_email='osmsupport@etsi.org', + maintainer='ETSI OSM', + maintainer_email='osmsupport@etsi.org', url='https://osm.etsi.org/gitweb/?p=osm/NBI.git;a=summary', license='Apache 2.0', packages=find_packages(exclude=["temp", "local"]), include_package_data=True, - # exclude_package_data={'': ['osm_nbi/local', 'temp']}, - # data_files=[('/etc/osm/', ['osm_nbi/nbi.cfg']), - # ('/etc/systemd/system/', ['osm_nbi/osm-nbi.service']), - # ], - install_requires=[ - 'CherryPy>=18.1.2', - 'osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common', - 'jsonschema>=3.2.0', - 'PyYAML', - 'osm-im @ git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im', - 'python-keystoneclient', - 'requests', - 'aiohttp>=2.3.10,<=3.6.2', - ], setup_requires=['setuptools-version-command'], ) diff --git a/stdeb.cfg b/stdeb.cfg index 8e4ea8a..074f69a 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -12,5 +12,3 @@ [DEFAULT] X-Python3-Version : >= 3.5 -Depends3 : python3-osm-common, python3-osm-im, python3-cherrypy3, python3-yaml, python3-jsonschema, - python3-keystoneclient, python3-pip, python3-requests, python3-aiohttp diff --git a/tox.ini b/tox.ini index 656be6c..a0fef5a 100644 --- a/tox.ini +++ b/tox.ini @@ -13,47 +13,111 @@ # See the License for the specific language governing permissions and # limitations under the License. + [tox] -envlist = cover, flake8 +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] usedevelop = True basepython = python3 -install_command = python3 -m pip install -r requirements.txt -U {opts} {packages} +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff osm_nbi/ + +####################################################################################### [testenv:cover] -basepython = python3 -deps = - nose2 - nose2-cov - coverage - -rrequirements.txt - -rtest-requirements.txt +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt commands = - coverage erase - nose2 -C --coverage osm_nbi --plugin nose2.plugins.junitxml -s osm_nbi/tests - coverage report --omit='*tests*' - coverage html -d ./cover --omit='*tests*' - coverage xml -o coverage.xml --omit='*tests*' + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osm_nbi -s osm_nbi/tests + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + +####################################################################################### [testenv:flake8] -basepython = python3 -deps = - flake8 - -rrequirements.txt +deps = flake8 commands = - flake8 osm_nbi setup.py --max-line-length 120 \ - --exclude .svn,CVS,.gz,.git,__pycache__,.tox,local,temp,osm_im,.temp-tox --ignore W291,W293,E226,E402,W504 + - flake8 osm_nbi/ setup.py -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command -commands = python3 setup.py --command-packages=stdeb.command bdist_deb -[testenv:unittest] -basepython = python3 -deps = asynctest - aioresponses -commands = python3 -m unittest discover osm_nbi/tests -v +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint +commands = + - pylint -E osm_nbi + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osm_nbi/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-nbi*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osm_nbi/requirements.txt' +whitelist_externals = sh +####################################################################################### +[flake8] +ignore = + W291, + W293, + W503, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, + test_mznmodels.py +max-line-length = 120 +show-source = True +builtins = _ \ No newline at end of file -- 2.25.1