From 5b55308c21032e222f1cc767e7670ad1c5db1272 Mon Sep 17 00:00:00 2001 From: beierlm Date: Tue, 16 Feb 2021 16:52:29 -0500 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Removes Makefile step and places all build logic into tox.ini Change-Id: I841f19700b462aed919cf7bc55f1b0a38728ef3e Signed-off-by: beierlm --- .gitignore | 6 + Dockerfile | 31 ++-- MANIFEST.in | 4 +- debian/python3-osm-pla.postinst | 29 ---- devops-stages/stage-build.sh | 4 +- devops-stages/stage-test.sh | 4 +- osm_pla/test/test_server.py | 5 +- requirements-dev.in | 16 ++ requirements-dev.txt | 32 ++++ requirements-dist.in | 17 ++ requirements-dist.txt | 23 +++ test-requirements.txt => requirements-test.in | 5 +- requirements-test.txt | 26 +++ Makefile => requirements.in | 16 +- requirements.txt | 43 ++--- setup.py | 17 +- stdeb.cfg | 2 +- tox.ini | 148 +++++++++++++----- 18 files changed, 300 insertions(+), 128 deletions(-) delete mode 100755 debian/python3-osm-pla.postinst create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements-dist.in create mode 100644 requirements-dist.txt rename test-requirements.txt => requirements-test.in (97%) create mode 100644 requirements-test.txt rename Makefile => requirements.in (68%) mode change 100755 => 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 9007d45..de32f53 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ #IDEs .idea .vscode +.project +.pydevproject #Generated folders when building the deb package locally .tox @@ -30,8 +32,12 @@ dists deb_dist pool dist +.coverage +cover +coverage.xml #local stuff, e.g. files that end in ".local" or folders called "local" *.local local venv + diff --git a/Dockerfile b/Dockerfile index 64e8f27..e6b2f78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -# Copyright 2020 ETSI -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,13 +10,30 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh +# FROM ubuntu:18.04 -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install git tox make python3 python3-pip python-all && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install python3-all debhelper python3-setuptools apt-utils libgl1-mesa-glx && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U setuptools setuptools-version-command stdeb +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 ADD https://github.com/MiniZinc/MiniZincIDE/releases/download/2.4.2/MiniZincIDE-2.4.2-bundle-linux-x86_64.tgz /minizinc.tgz @@ -31,7 +46,3 @@ RUN mkdir /entry_data \ ENV FZNEXEC "/entry_data/fzn-exec" ENV PATH "/minizinc/bin:${PATH}" - -RUN mkdir /placement -COPY ./osm_pla/test/pil_price_list.yaml /placement/. -COPY ./osm_pla/test/vnf_price_list.yaml /placement/. diff --git a/MANIFEST.in b/MANIFEST.in index 60da55a..9e642ee 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,7 +14,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -include requirements.txt + include README.md -recursive-include osm_pla *.py *.sh *.yaml *.j2 +recursive-include osm_pla *.py *.sh *.yaml *.j2 *.txt recursive-include devops-stages * diff --git a/debian/python3-osm-pla.postinst b/debian/python3-osm-pla.postinst deleted file mode 100755 index bed12db..0000000 --- a/debian/python3-osm-pla.postinst +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -## -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: OSM_TECH@list.etsi.org -## - -echo "POST INSTALL OSM-PLA" -# Currently it is not needed pip3 installation -# echo "Installing python dependencies via pip..." -# pip3 install pip==9.0.3 -# pip3 install --user aiokafka - -#Creation of log folder -mkdir -p /var/log/osm - -# systemctl enable osm-pla.service diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 9c2b16b..84e4819 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -12,4 +12,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -make +rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs + +tox -e dist \ No newline at end of file diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 69c6fcd..2d7b24e 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -11,4 +11,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -tox + +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/osm_pla/test/test_server.py b/osm_pla/test/test_server.py index 9d7582f..fdde3eb 100644 --- a/osm_pla/test/test_server.py +++ b/osm_pla/test/test_server.py @@ -12,6 +12,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + +# pylint: disable=E1120 + import asyncio import copy import os @@ -259,7 +262,7 @@ nsd_from_db = {"_id": "15fc1941-f095-4cd8-af2d-1000bd6d9eaa", 'mgmt-network': True, 'vim-network-name': 'lanretxe' }], - } + } ###################################################### diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..c458854 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,16 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git+https://osm.etsi.org/gerrit/osm/common.git@v9.0#egg=osm-common diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..816639a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,32 @@ +aiokafka==0.7.0 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +dataclasses==0.6 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +kafka-python==2.0.2 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # aiokafka +git+https://osm.etsi.org/gerrit/osm/common.git@v9.0#egg=osm-common + # via -r requirements-dev.in +pycrypto==2.6.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +pymongo==3.11.3 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +pyyaml==5.4.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/test-requirements.txt b/requirements-test.in similarity index 97% rename from test-requirements.txt rename to requirements-test.in index 429714f..9a0542d 100755 --- a/test-requirements.txt +++ b/requirements-test.in @@ -15,4 +15,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -coverage \ No newline at end of file + +coverage +mock +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..d0c984f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,26 @@ +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +mock==4.0.3 + # via -r requirements-test.in +nose2==0.10.0 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/Makefile b/requirements.in similarity index 68% rename from Makefile rename to requirements.in index 7e82f44..82788ed 100755 --- a/Makefile +++ b/requirements.in @@ -1,4 +1,4 @@ -# Copyright 2020 ArctosLabs Scandinava AB +# Copyright 2020 ArctosLabs Scandinavia AB # ************************************************************* # This file is part of OSM Placement module @@ -15,14 +15,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -## - -all: clean package - -clean: - rm -rf dist deb_dist osm_pla-*.tar.gz osm_pla.egg-info .eggs - -package: - python3 setup.py --command-packages=stdeb.command sdist_dsc - cp debian/python3-osm-pla.postinst deb_dist/osm-pla*/debian - cd deb_dist/osm-pla*/ && dpkg-buildpackage -rfakeroot -uc -us +pyyaml +pymzn==0.18.* +jinja2 diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 index 32d181e..6f8103b --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,24 @@ -# Copyright 2020 ArctosLabs Scandinavia AB -# ************************************************************* - -# This file is part of OSM Placement module -# All Rights Reserved to ArctosLabs Scandinavia AB - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - +jinja2==2.11.3 + # via -r requirements.in +markupsafe==1.1.1 + # via jinja2 +pymzn==0.18.3 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -pyyaml==5.1.2 -pymzn==0.18.* -jinja2==2.10.3 -git+https://osm.etsi.org/gerrit/osm/common.git@v9.0#egg=osm-common +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index 2c7cd04..bb062d7 100755 --- a/setup.py +++ b/setup.py @@ -23,10 +23,10 @@ def parse_requirements(requirements): _name = 'osm_pla' _version_command = ('git describe --match v* --tags --long --dirty', 'pep440-git-full') _description = 'OSM Placement Module' -_author = "Lars Goran Magnusson" -_author_email = 'lars-goran.magnusson@arctoslabs.com' -_maintainer = 'Lars Goran Magnusson' -_maintainer_email = 'lars-goran.magnusson@arctoslabs.com' +_author = "OSM Support" +_author_email = 'osmsupport@etsi.org' +_maintainer = 'OSM Support' +_maintainer_email = 'osmsupport@etsi.org' _license = 'Apache 2.0' _url = 'https://osm.etsi.org/gitweb?p=osm/PLA.git;a=tree' @@ -44,15 +44,6 @@ setup( license=_license, packages=[_name], package_dir={_name: _name}, - install_requires=[ - 'osm-common', - 'jinja2==2.10.3', - 'pymzn==0.18.3', - 'pyyaml==5.1.2' - ], - dependency_links=[ - 'git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common', - ], include_package_data=True, entry_points={ "console_scripts": [ diff --git a/stdeb.cfg b/stdeb.cfg index 05381db..017d71c 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -12,6 +12,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + [DEFAULT] X-Python3-Version : >= 3.5 -Depends3: python3-osm-common, python3-yaml, python3-jinja2, python3-pip diff --git a/tox.ini b/tox.ini index 68c6211..d244554 100644 --- a/tox.ini +++ b/tox.ini @@ -1,50 +1,124 @@ -## -# Copyright 2020 ArctosLabs Scandinavia AB +####################################################################################### +# Copyright ETSI Contributors and Others. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + [tox] -envlist = py3 +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] +usedevelop = True basepython = python3 -install_command = python3 -m pip install -U {opts} {packages} -deps = -rrequirements.txt - -rtest-requirements.txt -commands = python3 -m unittest discover -v +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt -[testenv:coverage] -basepython = python3 -deps = -rrequirements.txt - -rtest-requirements.txt -commands = coverage run -m unittest discover - coverage report --omit='*site-packages*','*test*','*__init__*' - coverage html -d ./.tox/coverage/HTMLreport --omit='*site-packages*','*test*','*__init__*' - coverage xml -o ./.tox/coverage/XMLreport/coverage.xml --omit='*site-packages*','*test*','*__init__*' - coverage erase +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff osm_pla/ + + +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osm_pla + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + +####################################################################################### [testenv:flake8] -basepython = python3 deps = flake8 - -rrequirements.txt -commands = flake8 {toxinidir}/osm_pla/ {toxinidir}/setup.py \ - --max-line-length 120 \ - --exclude test_mznmodels.py,.svn,CVS,.gz,.git,__pycache__,.tox,local,temp +commands = + flake8 osm_pla/ setup.py -[testenv:build] -basepython = python3 -# changedir ={toxinidir} -deps = stdeb - setuptools-version-command - -rrequirements.txt -commands = python3 setup.py --command-packages=stdeb.command bdist_deb + +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint +commands = + pylint -E osm_pla + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osm_pla/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-pla*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osm_pla/requirements.txt' +whitelist_externals = sh + +####################################################################################### +[flake8] +ignore = + W291, + W293, + W503, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, + test_mznmodels.py +max-line-length = 120 +show-source = True +builtins = _ -- 2.25.1