From 4e49c9652e85ef8fd6393759740e63628a04b97a Mon Sep 17 00:00:00 2001 From: beierlm Date: Thu, 11 Feb 2021 14:57:14 -0500 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Removes Makefile step and places all build logic into tox.ini Change-Id: I31129ce9de342595f7ad24603107697ddab831c4 Feature: 10296 Signed-off-by: beierlm --- .gitignore | 92 ++++++++++++++ .gitignore-common | 52 -------- ...hon3-osm-common.postinst => .gitlab-ci.yml | 24 +--- Dockerfile | 27 ++-- MANIFEST.in | 2 +- devops-stages/stage-build.sh | 5 +- devops-stages/stage-test.sh | 5 +- Makefile => nose2.cfg | 18 +-- requirements-dist.in | 17 +++ requirements-dist.txt | 23 ++++ requirements-test.in | 18 +++ requirements-test.txt | 40 ++++++ requirements.in | 20 +++ requirements.txt | 28 +++++ setup.py | 24 ++-- stdeb.cfg | 16 ++- tox.ini | 119 +++++++++++++----- 17 files changed, 383 insertions(+), 147 deletions(-) create mode 100644 .gitignore delete mode 100644 .gitignore-common rename debian/python3-osm-common.postinst => .gitlab-ci.yml (54%) mode change 100755 => 100644 rename Makefile => nose2.cfg (62%) create mode 100644 requirements-dist.in create mode 100644 requirements-dist.txt create mode 100644 requirements-test.in create mode 100644 requirements-test.txt create mode 100644 requirements.in create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a022457 --- /dev/null +++ b/.gitignore @@ -0,0 +1,92 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## +*.py[cod] + +# C extensions +*.so + +# log files +*.log + +# Packages +*.egg +*.egg-info +dist +build +.eggs +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +nohup.out + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +cover +coverage.xml +.tox +nosetests.xml +.testrepository +.venv +.cache + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? +.settings/ +__pycache__/ +.idea +*.db +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace +.vscode +.project +.pydevproject +deb_dist +*.tar.gz diff --git a/.gitignore-common b/.gitignore-common deleted file mode 100644 index 0019f5a..0000000 --- a/.gitignore-common +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a template with common files to be igonored, after clone make a copy to .gitignore -# cp .gitignore-common .gitignore - -*.pyc -*.pyo - -#auto-ignore -.gitignore - -#logs -logs - -#pycharm -.idea - -#eclipse -.project -.pydevproject -.settings - -#local stuff files that end in ".local" or folders called "local" -*.local -osm_common/local -osm_common/test/local - -#local stuff files that end in ".temp" or folders called "temp" -*.temp -osm_common/temp -osm_common/test/temp - -#distribution and package generation -build -dist -*.egg-info -*.egg-info -.eggs - diff --git a/debian/python3-osm-common.postinst b/.gitlab-ci.yml old mode 100755 new mode 100644 similarity index 54% rename from debian/python3-osm-common.postinst rename to .gitlab-ci.yml index fa43cbc..eb9750e --- a/debian/python3-osm-common.postinst +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ -#!/bin/bash - -## +# Copyright Contributors to ETSI OSM +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,20 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: OSM_TECH@list.etsi.org -## - -echo "POST INSTALL OSM-COMMON" -echo "Installing python dependencies via pip..." - -pip3 install dataclasses -pip3 install --upgrade aiokafka -pip3 install --upgrade pymongo -pip3 install --upgrade pyyaml==5.* -pip3 install pycrypto # added dependency python3-crypto seems not working - -#Creation of log folder -mkdir -p /var/log/osm +include: + - project: 'devops/cicd' + file: '/templates/osm-base.yml' diff --git a/Dockerfile b/Dockerfile index 43afa1c..3fad212 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,16 +10,27 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -# This Dockerfile is intented for devops and deb package generation +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Use Dockerfile.local for running osm/LCM in a docker container from source FROM ubuntu:18.04 -RUN apt-get update && apt-get -y install git make python python3 \ - libcurl4-gnutls-dev libgnutls28-dev tox python3-dev python3-pip \ - debhelper python-setuptools python-all python3-all apt-utils \ - python-magic && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb setuptools-version-command +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 diff --git a/MANIFEST.in b/MANIFEST.in index cbc5d1e..e8fc479 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,5 +17,5 @@ ## include README.rst -recursive-include osm_common *.py *.xml *.sh +recursive-include osm_common *.py *.xml *.sh *.txt recursive-include devops-stages * \ No newline at end of file diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index dfb7978..05b232e 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# moved to a Makefile in order to add post install. Needed for "pip3 install aiokafka", -# that is not available with a package +rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs .tox -make clean package +tox -e dist diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 2e7f488..e5a8e85 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -13,6 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -rm -f nosetests.xml -tox # flake8 unittest pytest cover - +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/Makefile b/nose2.cfg similarity index 62% rename from Makefile rename to nose2.cfg index ed4ad8d..6823267 100644 --- a/Makefile +++ b/nose2.cfg @@ -1,4 +1,5 @@ -# Copyright 2018 Telefonica S.A. +# -*- coding: utf-8 -*- +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: clean package +[unittest] +plugins = nose2.plugins.junitxml -clean: - rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs - -package: - python3 setup.py --command-packages=stdeb.command sdist_dsc - cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian - cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us +[junit-xml] +always-on = True +keep_restricted = False +path = nosetests.xml +test_fullname = False diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100644 index 0000000..0378739 --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,18 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +coverage +nose2 +pytest \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..134c015 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,40 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +attrs==21.2.0 + # via pytest +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +iniconfig==1.1.1 + # via pytest +nose2==0.10.0 + # via -r requirements-test.in +packaging==20.9 + # via pytest +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pyparsing==2.4.7 + # via packaging +pytest==6.2.4 + # via -r requirements-test.in +six==1.16.0 + # via nose2 +toml==0.10.2 + # via pytest diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..1ce6a65 --- /dev/null +++ b/requirements.in @@ -0,0 +1,20 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pymongo +aiokafka +pyyaml +pycrypto +dataclasses \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2ca19f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +aiokafka==0.7.0 + # via -r requirements.in +dataclasses==0.6 + # via -r requirements.in +kafka-python==2.0.2 + # via aiokafka +pycrypto==2.6.1 + # via -r requirements.in +pymongo==3.11.3 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index 157ffec..f53c839 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2018 Telefonica S.A. +# Copyright ETSI OSM Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,31 +25,21 @@ _name = "osm_common" README = open(os.path.join(here, 'README.rst')).read() setup( - # python_requires='>=3.5', name=_name, description='OSM common utilities', long_description=README, version_command=('git describe --tags --long --dirty --match v*', 'pep440-git-full'), - # version=VERSION, - # python_requires='>3.5', + author='ETSI OSM', - author_email='alfonso.tiernosepulveda@telefonica.com', - maintainer='Alfonso Tierno', - maintainer_email='alfonso.tiernosepulveda@telefonica.com', + author_email='osmsupport@etsi.com', + maintainer='ETSI OSM', + maintainer_email='osmsupport@etsi.com', + url='https://osm.etsi.org/gitweb/?p=osm/common.git;a=summary', license='Apache 2.0', setup_requires=['setuptools-version-command'], packages=[_name], include_package_data=True, - # scripts=['nbi.py'], - - # dataclasses is required by aiokafka for Python <3.7.0 - install_requires=[ - 'pymongo', - 'aiokafka', - 'PyYAML', - 'pycrypto', - 'dataclasses', - ], + ) diff --git a/stdeb.cfg b/stdeb.cfg index 5bd8819..c6a1b3a 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,3 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [DEFAULT] X-Python3-Version : >= 3.5 -Depends3 : python3-pip, python3-crypto diff --git a/tox.ini b/tox.ini index a7cf7b4..c3341ea 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,5 @@ -# Copyright 2018 Telefonica S.A. +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,53 +13,103 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +####################################################################################### [tox] -envlist = cover, flake8, unittest, pytest +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] usedevelop = True basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt -[testenv:cover] -basepython = python3 -deps = - nose2 - nose2-cov - coverage - pytest - pytest-asyncio - pycrypto +####################################################################################### +[testenv:black] +deps = black +skip_install = true commands = - coverage erase - nose2 -C --coverage osm_common --plugin nose2.plugins.junitxml -s osm_common/tests - coverage report --omit='*tests*' - coverage html -d ./cover --omit='*tests*' - coverage xml -o coverage.xml --omit='*tests*' + - black --check --diff osm_common/ -[testenv:pytest] -basepython = python3 -deps = pytest - pytest-asyncio - pycrypto -commands = pytest osm_common +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osm_common -s osm_common/tests + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh +####################################################################################### [testenv:flake8] -basepython = python3 deps = flake8 commands = - flake8 osm_common/ setup.py --max-line-length 120 --exclude .svn,CVS,.gz,.git,__pycache__,.tox,local,temp --ignore W291,W293,E226 + flake8 osm_common/ setup.py -[testenv:unittest] -basepython = python3 -deps = pycrypto - pytest -commands = python3 -m unittest discover osm_common.tests +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-test.txt + pylint +commands = + pylint -E osm_common -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command -commands = python3 setup.py --command-packages=stdeb.command bdist_deb +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osm_common/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osm_common/requirements.txt' +whitelist_externals = sh +####################################################################################### +[flake8] +ignore = + W291, + W293, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 +show-source = True +builtins = _ -- 2.25.1