From: David Garcia Date: Wed, 17 Mar 2021 14:28:47 +0000 (+0100) Subject: Feature 10296 X-Git-Tag: branch-sol006v331-start~23 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F19%2F10319%2F22;p=osm%2Fdevops.git Feature 10296 Update dockerfiles to fetch dependencies from Artifactory .deb and using pip3 -r instead of apt to resolve all upstream depenencies Update stage_3 to use URLs for artifactory packages instead of apt server for installation of packages Change-Id: I3abb1e1314862f386fb090cb18429f4283b0273d Signed-off-by: beierlm --- diff --git a/Dockerfile b/Dockerfile index ffc36135..c3297dac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,30 @@ # License for the specific language governing permissions and limitations # under the License. # -# For those usages not covered by the Apache License, Version 2.0 please -# contact: legal@canonical.com -# -# To get in touch with the maintainers, please contact: -# osm-charmers@lists.launchpad.net ## +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh +# FROM ubuntu:18.04 +ENV LANG en_IN.UTF-8 + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + dh-make \ + git \ + python3.8 \ + python3-all \ + python3-dev \ + python3-setuptools -RUN apt-get update && apt-get -y install git build-essential dh-make tox python python3.8 python3-dev python3-pip +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index bea54793..51b35139 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eux +set -eu CURRENT_DIR=`pwd` # Execute tests for charms @@ -21,6 +21,7 @@ CHARM_PATH="./installers/charm" CHARM_NAMES="keystone lcm mon nbi ng-ui pla pol prometheus ro grafana" for charm in $CHARM_NAMES; do cd $CHARM_PATH/$charm - tox --recreate + OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) + printf "$OUTPUT\n" cd $CURRENT_DIR done \ No newline at end of file diff --git a/docker/Keystone/Makefile b/docker/Keystone/Makefile deleted file mode 100644 index 2a4529e5..00000000 --- a/docker/Keystone/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: esousa@whitestack.com or glavado@whitestack.com -## - -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/LCM/Dockerfile b/docker/LCM/Dockerfile index 34e753a1..0038e78a 100644 --- a/docker/LCM/Dockerfile +++ b/docker/LCM/Dockerfile @@ -16,47 +16,24 @@ ######################################################################## -# This creates som/LCM docker from from last stable package - -FROM ubuntu:18.04 - -# Set the working directory to /app -WORKDIR /app/osm_lcm - -# avoid extra information from packages -RUN echo 'path-exclude /usr/share/doc/*\n\ -path-include /usr/share/doc/*/copyright\n\ -path-exclude /usr/share/man/*\n\ -path-exclude /usr/share/groff/*\n\ -path-exclude /usr/share/info/*\n\ -path-exclude /usr/share/lintian/*\n\ -path-exclude /usr/share/linda/*\n'\ -> /etc/dpkg/dpkg.cfg.d/01_nodoc && \ - echo 'APT::Install-Recommends "false";\n\ -APT::AutoRemove::RecommendsImportant "false";\n\ -APT::AutoRemove::SuggestsImportant "false";\n'\ -> /etc/apt/apt.conf.d/99_norecommends - -RUN apt-get update && apt-get -y install curl software-properties-common - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git make python3 \ - gpg-agent python3-pip \ - python3-all python3-setuptools openssh-client libffi-dev libssl-dev \ - wget tox python3-cffi \ - && python3 -m pip install -U pip \ - && python3 -m pip install -U juju==2.8.4 \ - && python3 -m pip install -U kubernetes==10.0.1 \ - && rm -rf /var/lib/apt/lists/* - -# RUN git clone https://osm.etsi.org/gerrit/osm/N2VC.git \ -# && cd N2VC \ -# && cd modules/libjuju && python3 setup.py develop && cd ../.. \ -# && python3 -m pip install -U -r requirements.txt - -RUN apt-get update && apt-get install -y curl xz-utils apt-transport-https \ - && curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ +FROM ubuntu:18.04 as INSTALL + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + apt-transport-https=1.6.12ubuntu0.2 \ + gnupg2=2.2.4-1ubuntu1.4 + +RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ - && apt-get update && apt-get install -y kubectl + && apt-get update && apt-get install -y kubectl=1.20.4-00 RUN curl https://get.helm.sh/helm-v2.15.2-linux-amd64.tar.gz --output helm-v2.15.2.tar.gz \ && tar -zxvf helm-v2.15.2.tar.gz \ @@ -72,30 +49,39 @@ RUN curl -L https://launchpad.net/juju/2.7/2.7.6/+download/juju-2.7.6-k8s.tar.xz && tar -xvf juju-2.7.6-k8s.tar.xz \ && mv juju /usr/local/bin/juju -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} LCM N2VC common" && apt update +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_LCM_URL +ARG PYTHON3_N2VC_URL + +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb + +RUN curl $PYTHON3_OSM_LCM_URL -o osm_lcm.deb +RUN dpkg -i ./osm_lcm.deb -RUN python3 -m pip install -U "cffi==1.13.2" +RUN curl $PYTHON3_N2VC_URL -o osm_n2vc.deb +RUN dpkg -i ./osm_n2vc.deb -ARG LCM_VERSION -ARG COMMON_VERSION -ARG N2VC_VERSION +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_lcm/requirements.txt \ + -r /usr/lib/python3/dist-packages/n2vc/requirements.txt -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - python3-osm-lcm${LCM_VERSION} \ - python3-osm-common${COMMON_VERSION} \ - python3-n2vc${N2VC_VERSION} \ - && rm -rf /var/lib/apt/lists/* +FROM ubuntu:18.04 + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 + +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages -LABEL Maintainer="alfonso.tiernosepulveda@telefonica.com" \ - Description="This implements life cycle management engine for OSM" \ - Version="1.0" \ - Author="Alfonso Tierno" +COPY --from=INSTALL /usr/bin/kubectl /usr/bin/kubectl +COPY --from=INSTALL /usr/local/bin/helm /usr/local/bin/helm +COPY --from=INSTALL /usr/local/bin/helm3 /usr/local/bin/helm3 +COPY --from=INSTALL /usr/local/bin/juju /usr/local/bin/juju + +######################################################################## # Used for local storage VOLUME /app/storage diff --git a/docker/LCM/Makefile b/docker/LCM/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/LCM/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/MON/Dockerfile b/docker/MON/Dockerfile index eefb5fbd..c25e6d2b 100644 --- a/docker/MON/Dockerfile +++ b/docker/MON/Dockerfile @@ -20,48 +20,43 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -FROM ubuntu:20.04 - -LABEL authors="Guillermo Calvino" - -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y curl \ - software-properties-common \ - git \ - python3 \ - python3-pip \ - libssl-dev \ - libffi-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && python3 -m pip install --upgrade pip -RUN python3 --version && pip --version - -RUN python3 -m pip install 'pyyaml>=5.1.2' 'paramiko' 'chardet==3.0.4' \ - 'lxml==4.6.2' 'humanfriendly==9.0.*' 'aiokafka>=0.7.0' \ - 'prometheus_client==0.4.*' 'python-keystoneclient==3.15.*' \ - 'six==1.15.0' 'requests==2.25.*' 'gnocchiclient==7.0.*' \ - 'pyvcloud==23.0.*' 'python-novaclient==12.0.*' \ - 'python-ceilometerclient==2.9.*' 'python-neutronclient==5.1.*' - -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing - -ARG MON_VERSION -ARG COMMON_VERSION -ARG N2VC_VERSION - -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - \ - && add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} MON N2VC common" \ - && apt update \ - && DEBIAN_FRONTEND=noninteractive apt-get -y install python3-osm-common${COMMON_VERSION} \ - python3-osm-mon${MON_VERSION} \ - python3-n2vc${N2VC_VERSION} - - -RUN python3 -m pip install -U "juju==2.8.4" +FROM ubuntu:20.04 as INSTALL + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + curl=7.68.0-1ubuntu2.4 \ + gcc=4:9.3.0-1ubuntu2 \ + python3=3.8.2-0ubuntu2 \ + python3-setuptools=45.2.0-1 \ + python3-dev=3.8.2-0ubuntu2 && \ + python3 -m easy_install pip==21.0.1 + +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_MON_URL +ARG PYTHON3_N2VC_URL + +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb + +RUN curl $PYTHON3_N2VC_URL -o n2vc.deb +RUN dpkg -i ./n2vc.deb + +RUN curl $PYTHON3_OSM_MON_URL -o osm_mon.deb +RUN dpkg -i ./osm_mon.deb + +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/n2vc/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_mon/requirements.txt + +FROM ubuntu:20.04 as FINAL + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.8.2-0ubuntu2 + +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages +COPY --from=INSTALL /usr/bin/osm* /usr/bin/ COPY scripts/ scripts/ diff --git a/docker/MON/Makefile b/docker/MON/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/MON/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index a3b40288..00000000 --- a/docker/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright 2017 Sandvine -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -SUBDIRS = MON NBI osmclient RO LCM light-ui POL Keystone NG-UI PLA tests -include mk/dirs.mk diff --git a/docker/NBI/Dockerfile b/docker/NBI/Dockerfile index c8ac78f8..d32c7120 100644 --- a/docker/NBI/Dockerfile +++ b/docker/NBI/Dockerfile @@ -16,43 +16,45 @@ ######################################################################## -# This creates osm/NBI docker from local NBI source code +FROM ubuntu:18.04 as INSTALL -FROM ubuntu:18.04 +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install curl software-properties-common +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_IM_URL +ARG PYTHON3_OSM_NBI_URL -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 python3-dev python3-pip +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb -RUN python3 -m pip install --upgrade pip +RUN curl $PYTHON3_OSM_IM_URL -o osm_im.deb +RUN dpkg -i ./osm_im.deb -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y rustc cargo +RUN curl $PYTHON3_OSM_NBI_URL -o osm_nbi.deb +RUN dpkg -i ./osm_nbi.deb -RUN python3 --version && python3 -m pip --version +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_im/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_nbi/requirements.txt -RUN rustc --version && cargo --version +FROM ubuntu:18.04 as FINAL -RUN python3 -m pip install --upgrade setuptools setuptools-rust +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 -RUN python3 -m pip install --upgrade aiokafka dataclasses python-keystoneclient pymongo jsonschema \ - aiohttp cherrypy==18.1.2 pyangbind keystoneauth1 \ - && mkdir -p /app/storage/kafka && mkdir -p /app/log +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing +RUN mkdir -p /app/storage/kafka && mkdir -p /app/log -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} NBI IM common" && apt update - -ARG NBI_VERSION -ARG COMMON_VERSION -ARG IM_VERSION - -RUN apt-get update &&DEBIAN_FRONTEND=noninteractive apt-get -y install python3-osm-nbi${NBI_VERSION} python3-osm-common${COMMON_VERSION} python3-osm-im${IM_VERSION} - -# Set the working directory to /app WORKDIR /app/osm_nbi EXPOSE 9999 @@ -61,11 +63,6 @@ RUN cp -R /usr/lib/python3/dist-packages/osm_nbi/html_public /app/osm_nbi/html_p RUN cp /usr/lib/python3/dist-packages/osm_nbi/nbi.cfg /app/osm_nbi/ RUN cp -R /usr/lib/python3/dist-packages/osm_nbi/http /app/osm_nbi/ -LABEL Maintainer="alfonso.tiernosepulveda@telefonica.com" \ - Description="This implements a north bound interface for OSM" \ - Version="1.0" \ - Author="Alfonso Tierno" - # Used for local storage VOLUME /app/storage # Used for logs diff --git a/docker/NBI/Makefile b/docker/NBI/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/NBI/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/NG-UI/Dockerfile b/docker/NG-UI/Dockerfile index 15e837f4..17416478 100644 --- a/docker/NG-UI/Dockerfile +++ b/docker/NG-UI/Dockerfile @@ -15,25 +15,24 @@ FROM ubuntu:18.04 -RUN apt-get update && apt-get install -y curl xz-utils gnupg2 \ - && apt-get update && apt-get install -y apt-transport-https software-properties-common \ - && curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ - && echo "deb https://deb.nodesource.com/node_10.x xenial main" | tee -a /etc/apt/sources.list.d/nodesource.list \ - && echo "deb-src https://deb.nodesource.com/node_10.x xenial main" | tee -a /etc/apt/sources.list.d/nodesource.list \ - && apt-get update && apt-get install -y nodejs \ - && apt-get install -y nginx - -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing -ARG NGUI_VERSION - -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} NG-UI" && apt-get update - -RUN apt-get install -y osm-ngui${NGUI_VERSION} && \ - rm -rf /var/lib/apt/lists/* +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + apt-transport-https=1.6.12ubuntu0.2 \ + curl=7.58.0-2ubuntu3.13 \ + gnupg2=2.2.4-1ubuntu1.4 \ + nginx=1.14.0-0ubuntu1.7 \ + software-properties-common=0.96.24.32.14 \ + xz-utils=5.2.2-1.3 + +RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo "deb https://deb.nodesource.com/node_10.x xenial main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -y nodejs=10.24.0-1nodesource1 + +ARG OSM_NGUI_URL + +RUN curl $OSM_NGUI_URL -o osm_ngui.deb +RUN dpkg -i ./osm_ngui.deb WORKDIR /usr/share/osm-ngui RUN npm install @@ -55,4 +54,3 @@ HEALTHCHECK --start-period=130s --interval=10s --timeout=5s --retries=12 \ # Starting server. CMD ["nginx", "-g", "daemon off;"] - diff --git a/docker/NG-UI/Makefile b/docker/NG-UI/Makefile deleted file mode 100644 index 0fb388be..00000000 --- a/docker/NG-UI/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -## -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## - -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/PLA/Dockerfile b/docker/PLA/Dockerfile index 8c2f97ff..f3c3eb19 100644 --- a/docker/PLA/Dockerfile +++ b/docker/PLA/Dockerfile @@ -13,40 +13,51 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:18.04 - -LABEL authors="Lars-Göran Magnusson" +FROM ubuntu:18.04 as INSTALL -RUN apt-get --yes update \ - &&DEBIAN_FRONTEND=noninteractive apt-get --yes install git python python-pip python3 python3-pip libmysqlclient-dev libssl-dev libffi-dev \ - &&DEBIAN_FRONTEND=noninteractive apt-get --yes install mysql-client curl software-properties-common libgl1-mesa-glx \ - && python3 -m pip install -U pip \ - && python3 -m pip install pymzn==0.18.3 +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_PLA_URL -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} PLA common" && apt update +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb -ARG PLA_VERSION -ARG COMMON_VERSION +RUN curl $PYTHON3_OSM_PLA_URL -o osm_pla.deb +RUN dpkg -i ./osm_pla.deb -RUN apt-get --yes update && DEBIAN_FRONTEND=noninteractive apt-get -y install python3-osm-common${COMMON_VERSION} python3-osm-pla${PLA_VERSION} - -COPY scripts/ scripts/ +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_pla/requirements.txt ADD https://github.com/MiniZinc/MiniZincIDE/releases/download/2.4.2/MiniZincIDE-2.4.2-bundle-linux-x86_64.tgz /minizinc.tgz RUN tar -zxf /minizinc.tgz && \ mv /MiniZincIDE-2.4.2-bundle-linux /minizinc +FROM ubuntu:18.04 +LABEL authors="Lars-Göran Magnusson" + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 + +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages +COPY --from=INSTALL /usr/bin/osm* /usr/bin/ +COPY --from=INSTALL /minizinc /minizinc + RUN mkdir /entry_data \ && mkdir /entry_data/mzn-lib \ && ln -s /entry_data/mzn-lib /minizinc/share/minizinc/exec +COPY scripts/ scripts/ RUN mkdir /placement ENV OSMPLA_MESSAGE_DRIVER kafka diff --git a/docker/PLA/Makefile b/docker/PLA/Makefile deleted file mode 100644 index 8763aad5..00000000 --- a/docker/PLA/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/POL/Dockerfile b/docker/POL/Dockerfile index f143b62a..0c25583f 100644 --- a/docker/POL/Dockerfile +++ b/docker/POL/Dockerfile @@ -20,30 +20,38 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -FROM ubuntu:20.04 +FROM ubuntu:20.04 as INSTALL -LABEL authors="Benjamín Díaz" +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + curl=7.68.0-1ubuntu2.4 \ + gcc=4:9.3.0-1ubuntu2 \ + python3=3.8.2-0ubuntu2 \ + python3-setuptools=45.2.0-1 \ + python3-dev=3.8.2-0ubuntu2 && \ + python3 -m easy_install pip==21.0.1 -RUN DEBIAN_FRONTEND=noninteractive apt-get -y update \ - && DEBIAN_FRONTEND=noninteractive apt-get -y install \ - git python3 python3-pip libmysqlclient-dev libssl-dev libffi-dev \ - mysql-client curl software-properties-common +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_POLICY_MODULE_URL -RUN pip3 install pip==21.0.1 +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing +RUN curl $PYTHON3_OSM_POLICY_MODULE_URL -o osm_policy_module.deb +RUN dpkg -i ./osm_policy_module.deb -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} POL common" && apt update +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_policy_module/requirements.txt -ARG POL_VERSION -ARG COMMON_VERSION +FROM ubuntu:20.04 as FINAL -RUN DEBIAN_FRONTEND=noninteractive apt-get -y update \ - && DEBIAN_FRONTEND=noninteractive apt-get -y install python3-osm-policy-module${POL_VERSION} +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.8.2-0ubuntu2 + +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages +COPY --from=INSTALL /usr/bin/osm* /usr/bin/ COPY scripts/ scripts/ diff --git a/docker/POL/Makefile b/docker/POL/Makefile deleted file mode 100644 index e7833c53..00000000 --- a/docker/POL/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) -CONTAINER_NAME=pol diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile index 09281bef..26c148b3 100644 --- a/docker/RO/Dockerfile +++ b/docker/RO/Dockerfile @@ -16,43 +16,77 @@ ######################################################################## -FROM ubuntu:18.04 - -MAINTAINER Gennadiy Dubina ; Alfonso Tierno - -RUN apt-get update && apt-get -y install curl software-properties-common git tox python3-pip \ - && python3 -m pip install --upgrade pip && python3 -m pip install pyangbind networking-l2gw - -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing - -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} RO IM common" && apt-get update +FROM ubuntu:18.04 as INSTALL + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 + +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + genisoimage=9:1.1.11-3ubuntu2 + +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_NG_RO_URL +ARG PYTHON3_OSM_RO_PLUGIN_URL +ARG PYTHON3_OSM_ROSDN_ARISTA_CLOUDVISION_URL +ARG PYTHON3_OSM_ROSDN_DPB_URL +ARG PYTHON3_OSM_ROSDN_DYNPAC_URL +ARG PYTHON3_OSM_ROSDN_FLOODLIGHTOF_URL +ARG PYTHON3_OSM_ROSDN_IETFL2VPN_URL +ARG PYTHON3_OSM_ROSDN_JUNIPER_CONTRAIL_URL +ARG PYTHON3_OSM_ROSDN_ODLOF_URL +ARG PYTHON3_OSM_ROSDN_ONOSOF_URL +ARG PYTHON3_OSM_ROSDN_ONOS_VPLS_URL +ARG PYTHON3_OSM_ROVIM_AWS_URL +ARG PYTHON3_OSM_ROVIM_AZURE_URL +ARG PYTHON3_OSM_ROVIM_FOS_URL +ARG PYTHON3_OSM_ROVIM_OPENNEBULA_URL +ARG PYTHON3_OSM_ROVIM_OPENSTACK_URL +ARG PYTHON3_OSM_ROVIM_OPENVIM_URL +ARG PYTHON3_OSM_ROVIM_VMWARE_URL + +RUN for URL in \ + $PYTHON3_OSM_COMMON_URL \ + $PYTHON3_OSM_NG_RO_URL \ + $PYTHON3_OSM_RO_PLUGIN_URL \ + $PYTHON3_OSM_ROSDN_ARISTA_CLOUDVISION_URL \ + $PYTHON3_OSM_ROSDN_DPB_URL \ + $PYTHON3_OSM_ROSDN_DYNPAC_URL \ + $PYTHON3_OSM_ROSDN_FLOODLIGHTOF_URL \ + $PYTHON3_OSM_ROSDN_IETFL2VPN_URL \ + $PYTHON3_OSM_ROSDN_JUNIPER_CONTRAIL_URL \ + $PYTHON3_OSM_ROSDN_ODLOF_URL \ + $PYTHON3_OSM_ROSDN_ONOSOF_URL \ + $PYTHON3_OSM_ROSDN_ONOS_VPLS_URL \ + $PYTHON3_OSM_ROVIM_AWS_URL \ + $PYTHON3_OSM_ROVIM_AZURE_URL \ + $PYTHON3_OSM_ROVIM_FOS_URL \ + $PYTHON3_OSM_ROVIM_OPENNEBULA_URL \ + $PYTHON3_OSM_ROVIM_OPENSTACK_URL \ + $PYTHON3_OSM_ROVIM_OPENVIM_URL \ + $PYTHON3_OSM_ROVIM_VMWARE_URL ; do \ + curl -s $URL -O ; \ + done + +RUN dpkg -i *.deb + +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_ng_ro/requirements.txt -ARG RO_VERSION -ARG COMMON_VERSION - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3-osm-common${COMMON_VERSION} \ - python3-osm-ro-plugin${RO_VERSION} python3-osm-ng-ro${RO_VERSION} \ - python3-osm-rosdn-arista-cloudvision${RO_VERSION} python3-osm-rosdn-dpb${RO_VERSION} \ - python3-osm-rosdn-dynpac${RO_VERSION} python3-osm-rosdn-floodlightof${RO_VERSION} \ - python3-osm-rosdn-ietfl2vpn${RO_VERSION} python3-osm-rosdn-juniper-contrail${RO_VERSION} \ - python3-osm-rosdn-odlof${RO_VERSION} python3-osm-rosdn-onos-vpls${RO_VERSION} \ - python3-osm-rosdn-onosof${RO_VERSION} \ - python3-osm-rovim-aws${RO_VERSION} python3-osm-rovim-azure${RO_VERSION} \ - python3-osm-rovim-fos${RO_VERSION} python3-osm-rovim-opennebula${RO_VERSION} \ - python3-osm-rovim-openstack${RO_VERSION} python3-osm-rovim-openvim${RO_VERSION} \ - python3-osm-rovim-vmware${RO_VERSION} - -# ensure right version is installed as python3-cryptography install an old version -# Since release 3.4 of python3-cryptography rust is an dependency +FROM ubuntu:18.04 -RUN DEBIAN_FRONTEND=noninteractive apt-get -y install build-essential libssl-dev libffi-dev \ - python3-dev cargo +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 -RUN python3 -m pip install --upgrade "cryptography>=2.5" +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages +COPY --from=INSTALL /usr/bin/genisoimage /usr/bin/genisoimage VOLUME /var/log/osm @@ -97,7 +131,6 @@ ENV OSMRO_LOG_LEVEL INFO HEALTHCHECK --start-period=130s --interval=10s --timeout=5s --retries=12 \ CMD curl --silent --fail http://localhost:9090/ro || exit 1 - # CMD curl --silent --fail localhost:9090/openmano/tenants || exit 1 -# CMD /usr/bin/RO-start.sh + CMD ["python3", "-u", "-m", "osm_ng_ro.ro_main"] diff --git a/docker/RO/Makefile b/docker/RO/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/RO/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/light-ui/Makefile b/docker/light-ui/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/light-ui/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/mk/Makefile.include b/docker/mk/Makefile.include deleted file mode 100644 index f31d2cae..00000000 --- a/docker/mk/Makefile.include +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -TOPDIR=$(shell readlink -f .|sed -e 's/\/docker\/.*//') -MKINCLUDE=$(TOPDIR)/docker/mk -MKBUILD=$(TOPDIR)/docker/build - -all: build - -TAG ?= 6 - -REPOSITORY_BASE ?= http://osm-download.etsi.org/repository/osm/debian -RELEASE ?= ReleaseNINE-daily -REPOSITORY_KEY ?= OSM%20ETSI%20Release%20Key.gpg -REPOSITORY ?= testing -NO_CACHE ?= --no-cache -DOCKER_REGISTRY ?= "" - -LOWER_MDG = $(shell echo $(MDG) | tr '[:upper:]' '[:lower:]') - -CONTAINER_NAME ?= $(LOWER_MDG) - -CMD_DOCKER_ARGS ?= -q -DOCKER_ARGS = $(CMD_DOCKER_ARGS) - -DEPS := MON IM LCM RO common osmclient devops NBI policy-module Keystone N2VC lightui ngui PLA tests - -DEPS_TARGETS = $(addprefix $(MKBUILD)/.dep_, $(DEPS)) - -Q=@ - -$(MKBUILD): - $Qmkdir -p $(MKBUILD) - -$(MKBUILD)/.dep_policy-module: - $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m POL -p policy-module > $@ - -$(MKBUILD)/.dep_lightui: - $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m LW-UI -p lightui > $@ - -$(MKBUILD)/.dep_ngui: - $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m NG-UI -p ngui > $@ - -$(MKBUILD)/.dep_%: - $Q$(MKINCLUDE)/get_version.sh -r $(REPOSITORY) -R $(RELEASE) -k $(REPOSITORY_KEY) -u $(REPOSITORY_BASE) -m $* > $@ - -build: $(MKBUILD) $(DEPS_TARGETS) - $Qdocker build -t opensourcemano/$(LOWER_MDG):$(TAG) \ - --build-arg RELEASE=$(RELEASE) \ - --build-arg REPOSITORY=$(REPOSITORY) \ - --build-arg REPOSITORY_KEY=$(REPOSITORY_KEY) \ - --build-arg REPOSITORY_BASE=$(REPOSITORY_BASE) \ - --build-arg MON_VERSION==$(shell cat $(MKBUILD)/.dep_MON) \ - --build-arg IM_VERSION==$(shell cat $(MKBUILD)/.dep_IM) \ - --build-arg RO_VERSION==$(shell cat $(MKBUILD)/.dep_RO) \ - --build-arg LCM_VERSION==$(shell cat $(MKBUILD)/.dep_LCM) \ - --build-arg COMMON_VERSION==$(shell cat $(MKBUILD)/.dep_common) \ - --build-arg OSMCLIENT_VERSION==$(shell cat $(MKBUILD)/.dep_osmclient) \ - --build-arg NBI_VERSION==$(shell cat $(MKBUILD)/.dep_NBI) \ - --build-arg POL_VERSION==$(shell cat $(MKBUILD)/.dep_policy-module) \ - --build-arg PLA_VERSION==$(shell cat $(MKBUILD)/.dep_PLA) \ - --build-arg DEVOPS_VERSION==$(shell cat $(MKBUILD)/.dep_devops) \ - --build-arg N2VC_VERSION==$(shell cat $(MKBUILD)/.dep_N2VC) \ - --build-arg LWUI_VERSION==$(shell cat $(MKBUILD)/.dep_lightui) \ - --build-arg NGUI_VERSION==$(shell cat $(MKBUILD)/.dep_ngui) \ - --build-arg TESTS_VERSION==$(shell cat $(MKBUILD)/.dep_tests) \ - --build-arg CACHE_DATE==$(shell date -uI) \ - $(DOCKER_ARGS) . - -clean: - rm -f $(MKBUILD)/.dep* - -tag: - docker tag opensourcemano/$(CONTAINER_NAME):$(INPUT_TAG) $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG) - -push: tag - docker push $(DOCKER_REGISTRY)opensourcemano/$(LOWER_MDG):$(TAG) diff --git a/docker/mk/dirs.mk b/docker/mk/dirs.mk deleted file mode 100644 index 0a3547ae..00000000 --- a/docker/mk/dirs.mk +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2017 Sandvine -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TOPDIR=$(shell readlink -f .|sed -e 's/\/descriptor-packages\/.*//') -TOOLS_DIR := $(TOPDIR)/tools - -SUBDIRS_CLEAN = $(addsuffix .clean, $(SUBDIRS)) -SUBDIRS_TEST = $(addsuffix .test, $(SUBDIRS)) -SUBDIRS_TAG = $(addsuffix .tag, $(SUBDIRS)) -SUBDIRS_PUSH = $(addsuffix .push, $(SUBDIRS)) - -.PHONY: $(SUBDIRS) $(SUBDIRS_CLEAN) clean test tag push - -all: $(SUBDIRS) - -clean: $(SUBDIRS_CLEAN) - -test: $(SUBDIRS_TEST) - -tag: $(SUBDIRS_TAG) - -push: $(SUBDIRS_PUSH) - -$(SUBDIRS_CLEAN): %.clean: - @$(MAKE) --no-print-directory -C $* clean - -$(SUBDIRS_TEST): %.test: - @$(MAKE) --no-print-directory -C $* test - -$(SUBDIRS_TAG): %.tag: - @$(MAKE) --no-print-directory -C $* tag - -$(SUBDIRS_PUSH): %.push: - @$(MAKE) --no-print-directory -C $* push - -$(SUBDIRS): - @$(MAKE) --no-print-directory -C $@ - -test: - $(TOOLS_DIR)/launch_tests.sh diff --git a/docker/mk/get_version.sh b/docker/mk/get_version.sh deleted file mode 100755 index 2fae03a5..00000000 --- a/docker/mk/get_version.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -# -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -RELEASE="ReleaseNINE-daily" -REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg" -REPOSITORY="testing" -REPOSITORY_BASE="http://osm-download.etsi.org/repository/osm/debian" -DEBUG= - -while getopts ":r:k:u:R:b:-:dm:p:" o; do - case "${o}" in - r) - REPOSITORY=${OPTARG} - ;; - R) - RELEASE=${OPTARG} - ;; - k) - REPOSITORY_KEY=${OPTARG} - ;; - u) - REPOSITORY_BASE=${OPTARG} - ;; - d) - DEBUG=y - ;; - p) - PACKAGE_NAME=${OPTARG} - ;; - m) - MDG=${OPTARG} - ;; - -) - ;; - esac -done - -if [ -z "$MDG" ]; then - echo "missing MDG" -fi - -[ -z "$PACKAGE_NAME" ] && PACKAGE_NAME=$MDG - -if [ -n "$DEBUG" ]; then - echo curl $REPOSITORY_BASE/$RELEASE/dists/$REPOSITORY/$MDG/binary-amd64/Packages -fi - -curl $REPOSITORY_BASE/$RELEASE/dists/$REPOSITORY/$MDG/binary-amd64/Packages 2>/dev/null | awk -v pkg=$PACKAGE_NAME '{ - if ( /Package:/ && match($2,sprintf("%s$",tolower(pkg)) ) ) { - package=1; - } else if (package==1 && match($1,"Version:")) { - package=0; - printf("%s\n", $2); - } -}' | head -1 diff --git a/docker/osmclient/Dockerfile b/docker/osmclient/Dockerfile index 68fa3e76..39b596d6 100644 --- a/docker/osmclient/Dockerfile +++ b/docker/osmclient/Dockerfile @@ -1,47 +1,54 @@ -## -# Copyright 2019 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # -# All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +FROM ubuntu:18.04 as INSTALL -FROM ubuntu:18.04 +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 -MAINTAINER Gerardo Garcia ; Jayant Madavi +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + libcurl4-openssl-dev=7.58.0-2ubuntu3.13 -RUN apt-get update && apt-get -y install curl software-properties-common +ARG PYTHON3_OSMCLIENT_URL +ARG PYTHON3_OSM_IM_URL -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install wget python3 python3-pip \ - libcurl4-openssl-dev libssl-dev python3-openstackclient +RUN curl $PYTHON3_OSMCLIENT_URL -o osmclient.deb +RUN dpkg -i ./osmclient.deb -RUN python3 -m pip install -U pip && python3 -m pip install pyyaml python-magic pyangbind pytest==4.6.3 verboselogs +RUN curl $PYTHON3_OSM_IM_URL -o osm_im.deb +RUN dpkg -i ./osm_im.deb -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osmclient/requirements.txt \ + -r /usr/lib/python3/dist-packages/osm_im/requirements.txt -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} devops osmclient IM" && apt-get update +FROM ubuntu:18.04 as FINAL -ARG OSMCLIENT_VERSION -ARG DEVOPS_VERSION -ARG IM_VERSION +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 -RUN apt-get update && apt-get -y install osm-devops${DEVOPS_VERSION} \ - python3-osm-im${IM_VERSION} python3-osmclient${OSMCLIENT_VERSION} -ENV OSM_SOL005=True -ENV OSM_HOSTNAME=nbi:9999 +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages +ENV OSM_SOL005=True +ENV OSM_HOSTNAME=nbi:9999 \ No newline at end of file diff --git a/docker/osmclient/Makefile b/docker/osmclient/Makefile deleted file mode 100644 index d64462ab..00000000 --- a/docker/osmclient/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile index 18ebe8e7..66da9faa 100644 --- a/docker/tests/Dockerfile +++ b/docker/tests/Dockerfile @@ -13,51 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:18.04 +######################################################################## -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install git software-properties-common \ - curl xz-utils gnupg2 apt-transport-https make python3 debhelper python3-setuptools python3-pip apt-utils \ - ssh iputils-ping libcurl4-openssl-dev libssl-dev python3-yaml \ - build-essential libssl-dev libffi-dev python3-dev -RUN add-apt-repository -y ppa:rmescandon/yq && apt update && apt install yq -y +FROM ubuntu:18.04 as INSTALL -RUN python3 -m pip install --upgrade pip +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc=4:7.4.0-1ubuntu2.3 \ + python3=3.6.7-1~18.04 \ + python3-dev=3.6.7-1~18.04 \ + python3-setuptools=39.0.1-2 \ + curl=7.58.0-2ubuntu3.13 && \ + python3 -m easy_install pip==21.0.1 -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y cargo rustc +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + git=1:2.17.1-1ubuntu0.8 \ + libcurl4-openssl-dev=7.58.0-2ubuntu3.13 \ + libssl-dev=1.1.1-1ubuntu2.1~18.04.9 -RUN python3 --version && python3 -m pip --version -RUN rustc --version && cargo --version +ARG OSM_TESTS_URL +ARG PYTHON3_OSM_IM_URL +ARG PYTHON3_OSMCLIENT_URL -RUN python3 -m pip install --upgrade setuptools setuptools-rust +RUN curl $PYTHON3_OSM_IM_URL -o osm_im.deb +RUN dpkg -i ./osm_im.deb -RUN python3 -m pip install --ignore-installed pyyaml +RUN curl $PYTHON3_OSMCLIENT_URL -o osmclient.deb +RUN dpkg -i ./osmclient.deb -RUN python3 -m pip install --upgrade haikunator requests robotframework robotframework-seleniumlibrary \ - robotframework-requests robotframework-jsonlibrary robotframework-sshlibrary charm-tools \ - verboselogs pyangbind python-openstackclient +RUN curl $OSM_TESTS_URL -o osm_tests.deb +RUN dpkg -i ./osm_tests.deb -ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian -ARG RELEASE=ReleaseNINE-daily -ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg -ARG REPOSITORY=testing -ARG TESTS_VERSION -ARG IM_VERSION -ARG OSMCLIENT_VERSION -RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} tests osmclient IM" && apt-get update +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_im/requirements.txt \ + -r /usr/lib/python3/dist-packages/osmclient/requirements.txt \ + -r /usr/share/osm-tests/requirements.txt -RUN apt-get install -y python3-osm-im${IM_VERSION} python3-osmclient${OSMCLIENT_VERSION} -RUN apt-get install -y osm-tests${TESTS_VERSION} && \ - rm -rf /var/lib/apt/lists/* +RUN git clone \ + https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git \ + --recurse-submodules \ + /robot-systest/osm-packages -RUN mv /usr/share/osm-tests/robot-systest /robot-systest -RUN mv /usr/share/osm-tests/charm.sh /usr/sbin/charm +FROM ubuntu:18.04 -WORKDIR /robot-systest +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install python3-minimal=3.6.7-1~18.04 -ARG CACHE_DATE=2021-03-01 -RUN git clone https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git --recurse-submodules /robot-systest/osm-packages +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.6/dist-packages /usr/local/lib/python3.6/dist-packages + +COPY --from=INSTALL /usr/share/osm-tests/robot-systest /robot-systest +COPY --from=INSTALL /usr/share/osm-tests/charm.sh /usr/sbin/charm +COPY --from=INSTALL /robot-systest/osm-packages /robot-systest/osm-packages + +WORKDIR /robot-systest # Folder where Robot tests are stored ENV ROBOT_DEVOPS_FOLDER=/robot-systest diff --git a/docker/tests/Makefile b/docker/tests/Makefile deleted file mode 100644 index 0fb388be..00000000 --- a/docker/tests/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -## -# Copyright 2020 ETSI -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## - -include ../mk/Makefile.include - -MDG=$(shell basename $(CURDIR)) diff --git a/jenkins/ci-pipelines/ci_helper.groovy b/jenkins/ci-pipelines/ci_helper.groovy index a2ededc5..f3430764 100644 --- a/jenkins/ci-pipelines/ci_helper.groovy +++ b/jenkins/ci-pipelines/ci_helper.groovy @@ -1,7 +1,7 @@ /* Copyright 2017 Sandvine * * All Rights Reserved. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. You may obtain * a copy of the License at @@ -35,10 +35,20 @@ def get_archive(artifactory_server, mdg, branch, build_name, build_number, patte ] }""" - server.download(downloadSpec) + def results = server.download(downloadSpec) + // Save the list of URLs that we need to pass to the dockerfiles for build + def debian_packages = [] + for ( result in results.getDependencies()) { + if (result.remotePath.contains(".deb")) { + debian_packages.add(result.remotePath) + } + } + // workaround. flatten repo to remove specific build num from the directory - sh "cp -R ${branch}/${build_num}/* ." - sh "rm -rf ${branch}/${build_num}" + sh "cp -R ${branch}/${build_number}/* ." + sh "rm -rf ${branch}/${build_number}" + + return debian_packages } def get_env_value(build_env_file,key) { @@ -70,10 +80,10 @@ def systest_run(container_name, test, source_rc = null) { so_ip = lxc_run(container_name,"lxc list SO-ub -c 4|grep eth0 |awk '{print \$2}'") ro_ip = lxc_run(container_name,"lxc list RO -c 4|grep eth0 |awk '{print \$2}'") //container_ip = get_ip_from_container(container_name) - + if ( source_rc ) { pre_source = "/tmp/" + source_rc.substring(source_rc.lastIndexOf('/')+1) - + lxc_file_push(container_name,source_rc,pre_source) result = lxc_run(container_name, "sh -c '. ${pre_source}; make -C devops/systest OSM_HOSTNAME=${so_ip} OSM_RO_HOSTNAME=${ro_ip} ${test}'") echo result @@ -157,7 +167,7 @@ def promote_build(artifactory_server,mdg,branch,buildInfo) { "buildName" : buildInfo.name, "buildNumber" : buildInfo.number, 'targetRepo' : 'osm-release', - + // Optional parameters 'comment' : 'this is the promotion comment', 'sourceRepo' : "${repo_prefix}${mdg}", diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index 5cad17b0..4edbd2bf 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -25,6 +25,7 @@ def project_checkout(url_prefix,project,refspec,revision) { if (GERRIT_PATCHSET_REVISION.size() > 0 ) { sh "git checkout -f ${revision}" } + sh "sudo git clean -dfx" } def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifactory_server,docker_args="",do_stage_4=false) { @@ -51,16 +52,18 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa container_name = "${project}-${branch}".toLowerCase() stage('Docker-Build') { - sh ''' - echo RUN groupadd -o -g $(id -g) -r jenkins >> Dockerfile - echo RUN useradd -o -u $(id -u) --create-home -r -g jenkins jenkins >> Dockerfile - ''' sh "docker build -t ${container_name} ." } - withDockerContainer(image: "${container_name}", args: docker_args) { + UID = sh(returnStdout:true, script: 'id -u').trim() + GID = sh(returnStdout:true, script: 'id -g').trim() + withDockerContainer(image: "${container_name}", args: docker_args + " -u root") { stage('Test') { - sh 'devops-stages/stage-test.sh' + sh "groupadd -o -g $GID -r jenkins" + sh "useradd -o -u $UID -d `pwd` -r -g jenkins jenkins" + sh "echo '#! /bin/sh' > /usr/bin/mesg" + sh "chmod 755 /usr/bin/mesg" + sh "runuser -l jenkins -c devops-stages/stage-test.sh" if (fileExists('coverage.xml')) { cobertura coberturaReportFile: 'coverage.xml' } @@ -69,12 +72,14 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } stage('Build') { - sh(returnStdout:true, script: 'devops-stages/stage-build.sh').trim() + sh(returnStdout:true, + script: "runuser -l $USER -c devops-stages/stage-build.sh").trim() } stage('Archive') { - sh "mkdir -p changelog" - sh "devops/tools/generatechangelog-pipeline.sh > changelog/changelog-${mdg}.html" - sh(returnStdout:true, script: 'devops-stages/stage-archive.sh').trim() + sh "runuser -l jenkins -c 'mkdir -p changelog'" + sh "runuser -l jenkins -c \"tools/generatechangelog-pipeline.sh > changelog/changelog-${mdg}.html\"" + sh(returnStdout:true, + script: "runuser -l jenkins -c devops-stages/stage-archive.sh").trim() ci_helper.archive(artifactory_server,mdg,branch,'untested') } } @@ -107,7 +112,6 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } if ( do_stage_3 ) { - stage('Build System') { def downstream_params_stage_3 = [ string(name: 'GERRIT_BRANCH', value: "${branch}"), @@ -129,7 +133,6 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa } } } - } return this diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index bdd9dd24..d7175b80 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -1,4 +1,4 @@ -/* Copyright 2017 Sandvine +/* Copyright ETSI Contributors and Others * * All Rights Reserved. * @@ -51,26 +51,10 @@ properties([ ]) ]) -def uninstall_osm(stackName) { - sh """ - export OSM_USE_LOCAL_DEVOPS=true - export PATH=$PATH:/snap/bin - installers/full_install_osm.sh -y -c swarm -w /tmp/osm -t ${stackName} -s ${stackName} --test --nolxd --nodocker --nojuju --nohostports --nohostclient --uninstall - """ -} - -def run_systest(stackName,tagName,testName,envfile=null) { - tempdir = sh(returnStdout: true, script: "mktemp -d").trim() - if ( !envfile ) - { - sh(script: "touch ${tempdir}/env") - envfile="${tempdir}/env" - } - sh "docker run --network net${stackName} --env-file ${envfile} -v ${tempdir}:/usr/share/osm-devops/systest/reports opensourcemano/osmclient:${tagName} make -C /usr/share/osm-devops/systest ${testName}" - sh "cp ${tempdir}/* ." - junit '*.xml' -} +//////////////////////////////////////////////////////////////////////////////////////// +// Helper Functions +//////////////////////////////////////////////////////////////////////////////////////// def run_robot_systest(tagName,testName,osmHostname,prometheusHostname,prometheus_port=null,envfile=null,kubeconfig=null,clouds=null,hostfile=null,jujuPassword=null) { tempdir = sh(returnStdout: true, script: "mktemp -d").trim() if ( !envfile ) @@ -156,6 +140,9 @@ def get_value(key, output) { } } +//////////////////////////////////////////////////////////////////////////////////////// +// Main Script +//////////////////////////////////////////////////////////////////////////////////////// node("${params.NODE}") { INTERNAL_DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/' @@ -187,123 +174,167 @@ node("${params.NODE}") { } container_name += "-${BUILD_NUMBER}" - // Copy the artifacts from the upstream jobs - stage("Copy Artifacts") { - // cleanup any previous repo - sh 'rm -rf repo' - dir("repo") { - // grab all stable upstream builds based on the - - dir("${RELEASE}") { - def list = ["RO", "openvim", "osmclient", "IM", "devops", "MON", "N2VC", "NBI", "common", "LCM", "POL", "LW-UI", "NG-UI", "PLA", "tests"] - for (component in list) { - step ([$class: 'CopyArtifact', - projectName: "${component}${upstream_main_job}/${GERRIT_BRANCH}"]) - - // grab the build name/number - build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER') - - // grab the archives from the stage_2 builds (ie. this will be the artifacts stored based on a merge) - ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${component}${upstream_main_job} :: ${GERRIT_BRANCH}", build_num) - - // cleanup any prevously defined dists - sh "rm -rf dists" - } - - // check if an upstream artifact based on specific build number has been requested - // This is the case of a merge build and the upstream merge build is not yet complete (it is not deemed - // a successful build yet). The upstream job is calling this downstream job (with the its build artifiact) - if ( params.UPSTREAM_JOB_NAME ) { - step ([$class: 'CopyArtifact', - projectName: "${params.UPSTREAM_JOB_NAME}", - selector: [$class: 'SpecificBuildSelector', buildNumber: "${params.UPSTREAM_JOB_NUMBER}"] - ]) - - build_num = ci_helper.get_env_value('build.env','BUILD_NUMBER') - component = ci_helper.get_mdg_from_project(ci_helper.get_env_value('build.env','GERRIT_PROJECT')) - - // the upstream job name contains suffix with the project. Need this stripped off - def project_without_branch = params.UPSTREAM_JOB_NAME.split('/')[0] - - // Remove the previous artifact for this component. Use the new upstream artifact - sh "rm -rf pool/${component}" + server_id = null + http_server_name = null + devopstempdir = null + useCharmedInstaller = params.INSTALLER.equalsIgnoreCase("charmed") - ci_helper.get_archive(params.ARTIFACTORY_SERVER,component,GERRIT_BRANCH, "${project_without_branch} :: ${GERRIT_BRANCH}", build_num) - - sh "rm -rf dists" - } + try { + builtModules = [:] +/////////////////////////////////////////////////////////////////////////////////////// +// Fetch stage 2 .deb artifacts +/////////////////////////////////////////////////////////////////////////////////////// + stage("Copy Artifacts") { + // cleanup any previous repo + sh 'rm -rf repo' + dir("repo") { + packageList = [] + dir("${RELEASE}") { + RELEASE_DIR = sh(returnStdout:true, script: 'pwd').trim() + + // check if an upstream artifact based on specific build number has been requested + // This is the case of a merge build and the upstream merge build is not yet complete (it is not deemed + // a successful build yet). The upstream job is calling this downstream job (with the its build artifiact) + def upstreamComponent="" + if ( params.UPSTREAM_JOB_NAME ) { + println("Fetching upstream job artifact from ${params.UPSTREAM_JOB_NAME}") + + step ([$class: 'CopyArtifact', + projectName: "${params.UPSTREAM_JOB_NAME}", + selector: [$class: 'SpecificBuildSelector', buildNumber: "${params.UPSTREAM_JOB_NUMBER}"] + ]) + + upstreamComponent = ci_helper.get_mdg_from_project(ci_helper.get_env_value('build.env','GERRIT_PROJECT')) + + // the upstream job name contains suffix with the project. Need this stripped off + def project_without_branch = params.UPSTREAM_JOB_NAME.split('/')[0] + def packages = ci_helper.get_archive(params.ARTIFACTORY_SERVER, + upstreamComponent, + GERRIT_BRANCH, + "${project_without_branch} :: ${GERRIT_BRANCH}", + ci_helper.get_env_value('build.env','BUILD_NUMBER')) + + packageList.addAll(packages) + println("${params.UPSTREAM_JOB_NAME}: ${packages}") + } - // sign all the components - for (component in list) { - sh "dpkg-sig --sign builder -k ${GPG_KEY_NAME} pool/${component}/*" + parallelSteps = [:] + def list = ["RO", "openvim", "osmclient", "IM", "devops", "MON", "N2VC", "NBI", "common", "LCM", "POL", "LW-UI", "NG-UI", "PLA", "tests"] + if (upstreamComponent.length()>0) { + println("Skipping upstream fetch of "+upstreamComponent) + list.remove(upstreamComponent) + } + for (buildStep in list) { + def component = buildStep + parallelSteps[component] = { + dir("$component") { + println("Fetching artifact for ${component}") + step ([$class: 'CopyArtifact', + projectName: "${component}${upstream_main_job}/${GERRIT_BRANCH}"]) + + // grab the archives from the stage_2 builds (ie. this will be the artifacts stored based on a merge) + def packages = ci_helper.get_archive(params.ARTIFACTORY_SERVER, + component, + GERRIT_BRANCH, + "${component}${upstream_main_job} :: ${GERRIT_BRANCH}", + ci_helper.get_env_value('build.env','BUILD_NUMBER')) + packageList.addAll(packages) + println("Fetched ${component}: ${packages}") + sh "rm -rf dists" + } + } + } + parallel parallelSteps + +/////////////////////////////////////////////////////////////////////////////////////// +// Create APT repository +/////////////////////////////////////////////////////////////////////////////////////// + for ( component in [ "devops" ] ) { + sh "mv ${component}/pool/${component} pool" + sh "dpkg-sig --sign builder -k ${GPG_KEY_NAME} pool/${component}/*" + sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/" + sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + } + // create and sign the release file + sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release" + sh "gpg --yes -abs -u ${GPG_KEY_NAME} -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release" + + // copy the public key into the release folder + // this pulls the key from the home dir of the current user (jenkins) + sh "cp ~/${REPO_KEY_NAME} 'OSM ETSI Release Key.gpg'" + sh "cp ~/${REPO_KEY_NAME} ." } - // now create the distro - for (component in list) { - sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/" - sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" - sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" - } + // start an apache server to serve up the packages + http_server_name = "${container_name}-apache" - // create and sign the release file - sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release" - sh "gpg --yes -abs -u ${GPG_KEY_NAME} -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release" - - // copy the public key into the release folder - // this pulls the key from the home dir of the current user (jenkins) - sh "cp ~/${REPO_KEY_NAME} 'OSM ETSI Release Key.gpg'" - sh "cp ~/${REPO_KEY_NAME} ." - - // merge the change logs - sh """ - rm -f changelog/changelog-osm.html - [ ! -d changelog ] || for mdgchange in \$(ls changelog); do cat changelog/\$mdgchange >> changelog/changelog-osm.html; done - """ - RELEASE_DIR = sh(returnStdout:true, script: 'pwd').trim() + pwd = sh(returnStdout:true, script: 'pwd').trim() + repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim() + repo_base_url = ci_helper.start_http_server(pwd,http_server_name,repo_port) + NODE_IP_ADDRESS=sh(returnStdout: true, script: + "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim() } - // start an apache server to serve up the images - http_server_name = "${container_name}-apache" - - pwd = sh(returnStdout:true, script: 'pwd').trim() - repo_port = sh(script: 'echo $(python -c \'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()\');', returnStdout: true).trim() - repo_base_url = ci_helper.start_http_server(pwd,http_server_name,repo_port) - NODE_IP_ADDRESS=sh(returnStdout: true, script: - "echo ${SSH_CONNECTION} | awk '{print \$3}'").trim() - } - - // now pull the devops package and install in temporary location - tempdir = sh(returnStdout: true, script: "mktemp -d").trim() - osm_devops_dpkg = sh(returnStdout: true, script: "find . -name osm-devops*.deb").trim() - sh "dpkg -x ${osm_devops_dpkg} ${tempdir}" - OSM_DEVOPS="${tempdir}/usr/share/osm-devops" - println("Repo base URL=${repo_base_url}") - } - - dir(OSM_DEVOPS) { - def remote = [:] - error = null - if ( params.DO_BUILD ) { - stage("Build") { - sh "make -C docker clean" - sh "make -C docker -j `nproc` Q= CMD_DOCKER_ARGS= TAG=${container_name} RELEASE=${params.RELEASE} REPOSITORY_BASE=${repo_base_url} REPOSITORY_KEY=${params.REPO_KEY_NAME} REPOSITORY=${params.REPO_DISTRO}" + // Unpack devops package into temporary location so that we use it from upstream if it was part of a patch + osm_devops_dpkg = sh(returnStdout: true, script: "find ./repo/release/pool/ -name osm-devops*.deb").trim() + devopstempdir = sh(returnStdout: true, script: "mktemp -d").trim() + println("Extracting local devops package ${osm_devops_dpkg} into ${devopstempdir} for docker build step") + sh "dpkg -x ${osm_devops_dpkg} ${devopstempdir}" + OSM_DEVOPS="${devopstempdir}/usr/share/osm-devops" + // Convert URLs from stage 2 packages to arguments that can be passed to docker build + for (remotePath in packageList) { + packageName=remotePath.substring(remotePath.lastIndexOf('/')+1) + packageName=packageName.substring(0,packageName.indexOf('_')) + builtModules[packageName]=remotePath } + } - stage("Push to internal registry") { +/////////////////////////////////////////////////////////////////////////////////////// +// Build docker containers +/////////////////////////////////////////////////////////////////////////////////////// + dir(OSM_DEVOPS) { + def remote = [:] + error = null + if ( params.DO_BUILD ) { withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'gitlab-registry', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) { sh "docker login ${INTERNAL_DOCKER_REGISTRY} -u ${USERNAME} -p ${PASSWORD}" } - sh "make -C docker push INPUT_TAG=${container_name} TAG=${container_name} DOCKER_REGISTRY=${INTERNAL_DOCKER_REGISTRY}" - } - - } - - try { - useCharmedInstaller = params.INSTALLER.equalsIgnoreCase("charmed") + moduleBuildArgs = "" + for (packageName in builtModules.keySet()) { + envName=packageName.replaceAll("-","_").toUpperCase()+"_URL" + moduleBuildArgs += " --build-arg ${envName}=" + builtModules[packageName] + } + dir ("docker") { + stage("Build") { + containerList = sh(returnStdout: true, script: + "find . -name Dockerfile -printf '%h\\n' | sed 's|\\./||'") + containerList=Arrays.asList(containerList.split("\n")) + print(containerList) + parallelSteps = [:] + for (buildStep in containerList) { + def module = buildStep + def moduleName = buildStep.toLowerCase() + def moduleTag = container_name + parallelSteps[module] = { + dir("$module") { + sh "docker build -t opensourcemano/${moduleName}:${moduleTag} ${moduleBuildArgs} ." + println("Tagging ${moduleName}:${moduleTag}") + sh "docker tag opensourcemano/${moduleName}:${moduleTag} ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}" + sh "docker push ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}" + } + } + } + parallel parallelSteps + } + } + } // if ( params.DO_BUILD ) if ( params.DO_INSTALL ) { - +/////////////////////////////////////////////////////////////////////////////////////// +// Launch VM +/////////////////////////////////////////////////////////////////////////////////////// stage("Spawn Remote VM") { println("Launching new VM") output=sh(returnStdout: true, script: """#!/bin/sh -e @@ -344,8 +375,11 @@ node("${params.NODE}") { alive = output.contains("succeeded") } println("VM is ready and accepting ssh connections") - } + } // stage("Spawn Remote VM") +/////////////////////////////////////////////////////////////////////////////////////// +// Installation +/////////////////////////////////////////////////////////////////////////////////////// stage("Install") { commit_id = '' repo_distro = '' @@ -396,7 +430,6 @@ node("${params.NODE}") { """ if ( useCharmedInstaller ) { - // Use local proxy for docker hub sshCommand remote: remote, command: ''' sudo snap install microk8s --classic --channel=1.19/stable @@ -438,10 +471,12 @@ node("${params.NODE}") { prometheusPort = 9091 osmHostname = IP_ADDRESS } - } - } + } // stage("Install") + } // if ( params.DO_INSTALL ) - stage_archive = false +/////////////////////////////////////////////////////////////////////////////////////// +// Health check of installed OSM in remote vm +/////////////////////////////////////////////////////////////////////////////////////// if ( params.DO_SMOKE ) { stage("OSM Health") { stackName = "osm" @@ -451,9 +486,11 @@ node("${params.NODE}") { } } +/////////////////////////////////////////////////////////////////////////////////////// +// Execute Robot tests +/////////////////////////////////////////////////////////////////////////////////////// + stage_archive = false if ( params.DO_STAGE_4 ) { - // override stage_archive to only archive on stable - stage_archive = false try { stage("System Integration Test") { if ( params.DO_ROBOT ) { @@ -485,7 +522,7 @@ EOF""" hostfile, jujuPassword) } - } + } // stage("System Integration Test") } finally { stage("Archive Container Logs") { // Archive logs to containers_logs.txt @@ -500,9 +537,8 @@ EOF""" } } } - } + } // if ( params.DO_STAGE_4 ) - // override to save the artifacts if ( params.SAVE_ARTIFACTS_OVERRIDE || stage_archive ) { stage("Archive") { sh "echo ${container_name} > build_version.txt" @@ -513,8 +549,20 @@ EOF""" ci_helper.archive(params.ARTIFACTORY_SERVER,RELEASE,GERRIT_BRANCH,'tested') } if ( params.DO_DOCKERPUSH ) { - stage("Docker Push") { - sh "make -C docker push INPUT_TAG=${container_name} TAG=${params.DOCKER_TAG}" + stage("Publish to Dockerhub") { + parallelSteps = [:] + for (buildStep in containerList) { + def module = buildStep + def moduleName = buildStep.toLowerCase() + def moduleTag = params.DOCKER_TAG + parallelSteps[module] = { + dir("$module") { + sh "docker tag opensourcemano/${moduleName}:${moduleTag} opensourcemano/${moduleName}:${moduleTag}" + sh "docker push opensourcemano/${moduleName}:${moduleTag}" + } + } + } + parallel parallelSteps } stage("Snap promotion") { @@ -544,41 +592,40 @@ EOF""" sh "snapcraft release $snap $edge_rev $beta_track" } } - } - } - } + } // stage("Snap promotion") + } // if ( params.DO_DOCKERPUSH ) + } // stage("Archive") + } // if ( params.SAVE_ARTIFACTS_OVERRIDE || stage_archive ) + } // dir(OSM_DEVOPS) + } finally { + if ( params.DO_INSTALL && server_id != null) { + delete_vm = true + if (error && params.SAVE_CONTAINER_ON_FAIL ) { + delete_vm = false + } + if (!error && params.SAVE_CONTAINER_ON_PASS ) { + delete_vm = false } - } - catch(Exception ex) { - error = ex - currentBuild.result = 'FAILURE' - println("Caught error: "+ex) - } - finally { - println("Entered finally block") - if ( params.DO_INSTALL && server_id != null) { - delete_vm = true - if (error && params.SAVE_CONTAINER_ON_FAIL ) { - delete_vm = false - } - if (!error && params.SAVE_CONTAINER_ON_PASS ) { - delete_vm = false - } - if ( delete_vm ) { - if (server_id != null) { - println("Deleting VM: $server_id") - sh """#!/bin/sh -e - for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done - openstack server delete ${server_id} - """ - } else { - println("Saved VM $server_id in ETSI VIM") - } + if ( delete_vm ) { + if (server_id != null) { + println("Deleting VM: $server_id") + sh """#!/bin/sh -e + for line in `grep OS ~/hive/robot-systest.cfg | grep -v OS_CLOUD` ; do export \$line ; done + openstack server delete ${server_id} + """ + } else { + println("Saved VM $server_id in ETSI VIM") } } + } + if ( http_server_name != null ) { sh "docker stop ${http_server_name} || true" sh "docker rm ${http_server_name} || true" } + + if ( devopstempdir != null ) { + sh "rm -rf ${devopstempdir}" + } } }