From: beierlm Date: Thu, 11 Feb 2021 19:57:14 +0000 (-0500) Subject: Feature 10296 Pip Standardization X-Git-Tag: v9.1.3 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=52db98d8854374247876058a647c6130a83559df;p=osm%2Fcommon.git Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Removes Makefile step and places all build logic into tox.ini Change-Id: I31129ce9de342595f7ad24603107697ddab831c4 Feature: 10296 Signed-off-by: beierlm --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a022457 --- /dev/null +++ b/.gitignore @@ -0,0 +1,92 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## +*.py[cod] + +# C extensions +*.so + +# log files +*.log + +# Packages +*.egg +*.egg-info +dist +build +.eggs +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +nohup.out + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +cover +coverage.xml +.tox +nosetests.xml +.testrepository +.venv +.cache + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? +.settings/ +__pycache__/ +.idea +*.db +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace +.vscode +.project +.pydevproject +deb_dist +*.tar.gz diff --git a/.gitignore-common b/.gitignore-common deleted file mode 100644 index 0019f5a..0000000 --- a/.gitignore-common +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a template with common files to be igonored, after clone make a copy to .gitignore -# cp .gitignore-common .gitignore - -*.pyc -*.pyo - -#auto-ignore -.gitignore - -#logs -logs - -#pycharm -.idea - -#eclipse -.project -.pydevproject -.settings - -#local stuff files that end in ".local" or folders called "local" -*.local -osm_common/local -osm_common/test/local - -#local stuff files that end in ".temp" or folders called "temp" -*.temp -osm_common/temp -osm_common/test/temp - -#distribution and package generation -build -dist -*.egg-info -*.egg-info -.eggs - diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..eb9750e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Copyright Contributors to ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +include: + - project: 'devops/cicd' + file: '/templates/osm-base.yml' diff --git a/Dockerfile b/Dockerfile index 43afa1c..3fad212 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,16 +10,27 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -# This Dockerfile is intented for devops and deb package generation +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Use Dockerfile.local for running osm/LCM in a docker container from source FROM ubuntu:18.04 -RUN apt-get update && apt-get -y install git make python python3 \ - libcurl4-gnutls-dev libgnutls28-dev tox python3-dev python3-pip \ - debhelper python-setuptools python-all python3-all apt-utils \ - python-magic && \ - DEBIAN_FRONTEND=noninteractive pip3 install -U stdeb setuptools-version-command +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 diff --git a/MANIFEST.in b/MANIFEST.in index cbc5d1e..e8fc479 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,5 +17,5 @@ ## include README.rst -recursive-include osm_common *.py *.xml *.sh +recursive-include osm_common *.py *.xml *.sh *.txt recursive-include devops-stages * \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index ed4ad8d..0000000 --- a/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2018 Telefonica S.A. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -all: clean package - -clean: - rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs - -package: - python3 setup.py --command-packages=stdeb.command sdist_dsc - cp debian/python3-osm-common.postinst deb_dist/osm-common*/debian - cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us diff --git a/debian/python3-osm-common.postinst b/debian/python3-osm-common.postinst deleted file mode 100755 index fa43cbc..0000000 --- a/debian/python3-osm-common.postinst +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -## -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: OSM_TECH@list.etsi.org -## - -echo "POST INSTALL OSM-COMMON" -echo "Installing python dependencies via pip..." - -pip3 install dataclasses -pip3 install --upgrade aiokafka -pip3 install --upgrade pymongo -pip3 install --upgrade pyyaml==5.* -pip3 install pycrypto # added dependency python3-crypto seems not working - -#Creation of log folder -mkdir -p /var/log/osm - diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index dfb7978..05b232e 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# moved to a Makefile in order to add post install. Needed for "pip3 install aiokafka", -# that is not available with a package +rm -rf dist deb_dist osm_common-*.tar.gz osm_common.egg-info .eggs .tox -make clean package +tox -e dist diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 2e7f488..e5a8e85 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -13,6 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -rm -f nosetests.xml -tox # flake8 unittest pytest cover - +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/nose2.cfg b/nose2.cfg new file mode 100644 index 0000000..6823267 --- /dev/null +++ b/nose2.cfg @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[unittest] +plugins = nose2.plugins.junitxml + +[junit-xml] +always-on = True +keep_restricted = False +path = nosetests.xml +test_fullname = False diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100644 index 0000000..0378739 --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,18 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +coverage +nose2 +pytest \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..134c015 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,40 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +attrs==21.2.0 + # via pytest +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +iniconfig==1.1.1 + # via pytest +nose2==0.10.0 + # via -r requirements-test.in +packaging==20.9 + # via pytest +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pyparsing==2.4.7 + # via packaging +pytest==6.2.4 + # via -r requirements-test.in +six==1.16.0 + # via nose2 +toml==0.10.2 + # via pytest diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..1ce6a65 --- /dev/null +++ b/requirements.in @@ -0,0 +1,20 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pymongo +aiokafka +pyyaml +pycrypto +dataclasses \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2ca19f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +aiokafka==0.7.0 + # via -r requirements.in +dataclasses==0.6 + # via -r requirements.in +kafka-python==2.0.2 + # via aiokafka +pycrypto==2.6.1 + # via -r requirements.in +pymongo==3.11.3 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index 157ffec..f53c839 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2018 Telefonica S.A. +# Copyright ETSI OSM Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,31 +25,21 @@ _name = "osm_common" README = open(os.path.join(here, 'README.rst')).read() setup( - # python_requires='>=3.5', name=_name, description='OSM common utilities', long_description=README, version_command=('git describe --tags --long --dirty --match v*', 'pep440-git-full'), - # version=VERSION, - # python_requires='>3.5', + author='ETSI OSM', - author_email='alfonso.tiernosepulveda@telefonica.com', - maintainer='Alfonso Tierno', - maintainer_email='alfonso.tiernosepulveda@telefonica.com', + author_email='osmsupport@etsi.com', + maintainer='ETSI OSM', + maintainer_email='osmsupport@etsi.com', + url='https://osm.etsi.org/gitweb/?p=osm/common.git;a=summary', license='Apache 2.0', setup_requires=['setuptools-version-command'], packages=[_name], include_package_data=True, - # scripts=['nbi.py'], - - # dataclasses is required by aiokafka for Python <3.7.0 - install_requires=[ - 'pymongo', - 'aiokafka', - 'PyYAML', - 'pycrypto', - 'dataclasses', - ], + ) diff --git a/stdeb.cfg b/stdeb.cfg index 5bd8819..c6a1b3a 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,3 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [DEFAULT] X-Python3-Version : >= 3.5 -Depends3 : python3-pip, python3-crypto diff --git a/tox.ini b/tox.ini index a7cf7b4..c3341ea 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,5 @@ -# Copyright 2018 Telefonica S.A. +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,53 +13,103 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +####################################################################################### [tox] -envlist = cover, flake8, unittest, pytest +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] usedevelop = True basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt -[testenv:cover] -basepython = python3 -deps = - nose2 - nose2-cov - coverage - pytest - pytest-asyncio - pycrypto +####################################################################################### +[testenv:black] +deps = black +skip_install = true commands = - coverage erase - nose2 -C --coverage osm_common --plugin nose2.plugins.junitxml -s osm_common/tests - coverage report --omit='*tests*' - coverage html -d ./cover --omit='*tests*' - coverage xml -o coverage.xml --omit='*tests*' + - black --check --diff osm_common/ -[testenv:pytest] -basepython = python3 -deps = pytest - pytest-asyncio - pycrypto -commands = pytest osm_common +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osm_common -s osm_common/tests + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh +####################################################################################### [testenv:flake8] -basepython = python3 deps = flake8 commands = - flake8 osm_common/ setup.py --max-line-length 120 --exclude .svn,CVS,.gz,.git,__pycache__,.tox,local,temp --ignore W291,W293,E226 + flake8 osm_common/ setup.py -[testenv:unittest] -basepython = python3 -deps = pycrypto - pytest -commands = python3 -m unittest discover osm_common.tests +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-test.txt + pylint +commands = + pylint -E osm_common -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command -commands = python3 setup.py --command-packages=stdeb.command bdist_deb +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osm_common/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-common*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osm_common/requirements.txt' +whitelist_externals = sh +####################################################################################### +[flake8] +ignore = + W291, + W293, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 +show-source = True +builtins = _