From e7646259a1e3047d089fc9b8ca023475288ce533 Mon Sep 17 00:00:00 2001 From: beierlm Date: Thu, 13 Jan 2022 10:53:08 -0500 Subject: [PATCH] Bug 1858: Adding APT proxy Adds apt proxy to build scripts and dockerfiles. Backward compatible change and proxy will not be used unless environment variable is supplied to docker build command. Fixes bug 1858 Change-Id: Ibd25bd1c145b8d43eeb85c5c952aec30ebcce1fc Signed-off-by: beierlm --- docker/Keystone/Dockerfile | 32 +++++++++++--------- docker/LCM/Dockerfile | 31 +++++++++++-------- docker/MON/Dockerfile | 41 +++++++++++++------------- docker/NBI/Dockerfile | 31 +++++++++++-------- docker/NG-UI/Dockerfile | 10 ++++++- docker/PLA/Dockerfile | 10 ++++++- docker/POL/Dockerfile | 41 +++++++++++++------------- docker/Prometheus/Dockerfile | 35 ++++++++++------------ docker/RO/Dockerfile | 31 +++++++++++-------- docker/osmclient/Dockerfile | 6 ++++ docker/tests/Dockerfile | 12 ++++++-- jenkins/ci-pipelines/ci_stage_2.groovy | 3 +- jenkins/ci-pipelines/ci_stage_3.groovy | 3 +- tools/local-build.sh | 16 ++++++++-- 14 files changed, 180 insertions(+), 122 deletions(-) diff --git a/docker/Keystone/Dockerfile b/docker/Keystone/Dockerfile index e81a7b57..9c0dc1ed 100644 --- a/docker/Keystone/Dockerfile +++ b/docker/Keystone/Dockerfile @@ -1,23 +1,27 @@ -# Copyright 2021 Whitestack, LLC +####################################################################################### +# Copyright ETSI Contributors and Others. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: fbravo@whitestack.com or glavado@whitestack.com -## - +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:20.04 +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + EXPOSE 5000 WORKDIR /keystone diff --git a/docker/LCM/Dockerfile b/docker/LCM/Dockerfile index 370f1fe0..15cf65ab 100644 --- a/docker/LCM/Dockerfile +++ b/docker/LCM/Dockerfile @@ -1,23 +1,28 @@ -## -# Copyright 2019 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## - -######################################################################## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/docker/MON/Dockerfile b/docker/MON/Dockerfile index a9b95fc0..adb736ac 100644 --- a/docker/MON/Dockerfile +++ b/docker/MON/Dockerfile @@ -1,27 +1,28 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:20.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ curl=7.68.* \ diff --git a/docker/NBI/Dockerfile b/docker/NBI/Dockerfile index 2e69264a..3c4a7f19 100644 --- a/docker/NBI/Dockerfile +++ b/docker/NBI/Dockerfile @@ -1,23 +1,28 @@ -## -# Copyright 2019 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## - -######################################################################## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/docker/NG-UI/Dockerfile b/docker/NG-UI/Dockerfile index 53ec2a9d..8350034c 100644 --- a/docker/NG-UI/Dockerfile +++ b/docker/NG-UI/Dockerfile @@ -1,4 +1,5 @@ -# Copyright 2020 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +13,16 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +####################################################################################### FROM ubuntu:18.04 AS BUILD +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ apt-transport-https=1.6.* \ diff --git a/docker/PLA/Dockerfile b/docker/PLA/Dockerfile index b673cb0b..dde6a373 100644 --- a/docker/PLA/Dockerfile +++ b/docker/PLA/Dockerfile @@ -1,4 +1,5 @@ -# Copyright 2020 Arctos Labs Scandinavia AB +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +13,16 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +####################################################################################### FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/docker/POL/Dockerfile b/docker/POL/Dockerfile index ee456ca3..0046183e 100644 --- a/docker/POL/Dockerfile +++ b/docker/POL/Dockerfile @@ -1,27 +1,28 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:20.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ curl=7.68.* \ diff --git a/docker/Prometheus/Dockerfile b/docker/Prometheus/Dockerfile index b32a95d8..7c842077 100644 --- a/docker/Prometheus/Dockerfile +++ b/docker/Prometheus/Dockerfile @@ -1,24 +1,19 @@ -# Copyright 2020 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: fbravo@whitestack.com or glavado@whitestack.com -## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM python:3.8 diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile index 39462de6..b2f32a4c 100644 --- a/docker/RO/Dockerfile +++ b/docker/RO/Dockerfile @@ -1,23 +1,28 @@ -## -# Copyright 2019 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -## - -######################################################################## +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/docker/osmclient/Dockerfile b/docker/osmclient/Dockerfile index 299990ad..49b01f37 100644 --- a/docker/osmclient/Dockerfile +++ b/docker/osmclient/Dockerfile @@ -17,6 +17,12 @@ FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile index 4874ef1e..4f7ef7c0 100644 --- a/docker/tests/Dockerfile +++ b/docker/tests/Dockerfile @@ -1,4 +1,5 @@ -# Copyright 2020 ETSI +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,11 +13,16 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -######################################################################## +####################################################################################### FROM ubuntu:18.04 as INSTALL +ARG APT_PROXY +RUN if [ ! -z $APT_PROXY ] ; then \ + echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\ + echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\ + fi + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc=4:7.4.* \ diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index dd9cc9b1..5c2267ca 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -52,7 +52,8 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa container_name = "${project}-${branch}".toLowerCase() stage('Docker-Build') { - sh "docker build -t ${container_name} ." + APT_PROXY = "http://172.21.1.1:3142" + sh "docker build --build-arg APT_PROXY=${APT_PROXY} -t ${container_name} ." } UID = sh(returnStdout:true, script: 'id -u').trim() diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index cd7cc8ed..0dbeb7dc 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -161,6 +161,7 @@ node("${params.NODE}") { INTERNAL_DOCKER_REGISTRY = 'osm.etsi.org:5050/devops/cicd/' INTERNAL_DOCKER_PROXY = 'http://172.21.1.1:5000' + APT_PROXY="http://172.21.1.1:3142" SSH_KEY = '~/hive/cicd_rsa' sh 'env' @@ -341,7 +342,7 @@ node("${params.NODE}") { def moduleTag = container_name parallelSteps[module] = { dir("$module") { - sh "docker build -t opensourcemano/${moduleName}:${moduleTag} ${moduleBuildArgs} ." + sh "docker build --build-arg APT_PROXY=${APT_PROXY} -t opensourcemano/${moduleName}:${moduleTag} ${moduleBuildArgs} ." println("Tagging ${moduleName}:${moduleTag}") sh "docker tag opensourcemano/${moduleName}:${moduleTag} ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}" sh "docker push ${INTERNAL_DOCKER_REGISTRY}opensourcemano/${moduleName}:${moduleTag}" diff --git a/tools/local-build.sh b/tools/local-build.sh index 08985181..9c402722 100755 --- a/tools/local-build.sh +++ b/tools/local-build.sh @@ -16,6 +16,7 @@ # limitations under the License. ####################################################################################### +APT_PROXY="" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" HTTPDDIR="$( cd "${HOME}/snap/qhttp/common" &> /dev/null && pwd )" HTTPPORT=8000 @@ -30,6 +31,7 @@ function check_arguments(){ while [ $# -gt 0 ] ; do case $1 in --debug) set -x ;; + --apt-proxy) APT_PROXY="$2" && shift ;; --help | -h) show_help && exit 0 ;; --httpddir) HTTPDIR="$2" && shift;; --install-local-registry) 'install_local_registry' ;; @@ -67,8 +69,9 @@ same process as Jenkins. OPTIONS: --help display this help message + --apt-proxy provide an apt proxy to docker build steps --debug enable set -x for this script - --install-local-registry install and enable Microk8s local registry on port 32000 + --install-local-registry install and enable Microk8s local registry on port 32000 --install-microstack install Microstack and configure to run robot tests --install-qhttpd install QHTTPD as an HTTP server on port ${HTTPPORT} --kubecfg path to kubecfg.yaml (uses Charmed OSM by default) @@ -171,7 +174,12 @@ function stage_2() { print_section "Building ${MODULE}" cd ${MODULE} find . -name '*.deb' -exec rm -v {} \; - docker build ${NO_CACHE} -t ${MODULE,,}-stage2 . + + BUILD_ARGS="" + if [ ! -z $APT_PROXY ] ; then + BUILD_ARGS="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} " + fi + docker build ${NO_CACHE} ${BUILD_ARGS} -t ${MODULE,,}-stage2 . STAGES="stage-build.sh" if [ ! -z $TESTS ] ; then @@ -217,6 +225,10 @@ function stage_3() { print_section "Performing Stage 3" MODULES=$(_find_module_dockerfile $1) BUILD_ARGS="" + if [ ! -z $APT_PROXY ] ; then + BUILD_ARGS="${BUILD_ARGS}--build-arg APT_PROXY=${APT_PROXY} " + fi + HOSTIP=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+') for file in ~/snap/qhttp/common/*.deb ; do file=`basename ${file}` -- 2.17.1