From b530189abd573f064af1b00973caf07bdcb9994f Mon Sep 17 00:00:00 2001 From: David Garcia Date: Mon, 14 Dec 2020 17:04:24 +0100 Subject: [PATCH] Fix basic-12: ns primitive packages --- nscharm_ns/charms/ns/.build.manifest | 61 +++-- .../charms/ns/.travis/profile-update.yaml | 12 + nscharm_ns/charms/ns/bin/charm-env | 6 +- nscharm_ns/charms/ns/layer.yaml | 2 +- .../charms/ns/lib/charms/layer/basic.py | 258 ++++++++++++++---- nscharm_ns/charms/ns/reactive/ns.py | 5 +- nscharm_ns/charms/ns/requirements.txt | 1 + nscharm_ns/charms/ns/version | 2 +- nscharm_ns/charms/ns/wheelhouse.txt | 16 ++ nscharm_ns/nscharm_nsd.yaml | 4 +- .../charms/vnf-policy/.build.manifest | 63 +++-- .../vnf-policy/.travis/profile-update.yaml | 12 + .../charms/vnf-policy/bin/charm-env | 6 +- .../charms/vnf-policy/layer.yaml | 2 +- .../vnf-policy/lib/charms/layer/basic.py | 258 ++++++++++++++---- .../charms/vnf-policy/requirements.txt | 1 + .../charms/vnf-policy/wheelhouse.txt | 36 +++ .../charms/vnf-user/.build.manifest | 63 +++-- .../vnf-user/.travis/profile-update.yaml | 12 + .../charms/vnf-user/bin/charm-env | 6 +- nscharm_user_vnf/charms/vnf-user/layer.yaml | 2 +- .../charms/vnf-user/lib/charms/layer/basic.py | 258 ++++++++++++++---- .../charms/vnf-user/requirements.txt | 1 + .../charms/vnf-user/wheelhouse.txt | 36 +++ 24 files changed, 889 insertions(+), 234 deletions(-) create mode 100644 nscharm_ns/charms/ns/.travis/profile-update.yaml create mode 100644 nscharm_ns/charms/ns/wheelhouse.txt create mode 100644 nscharm_policy_vnf/charms/vnf-policy/.travis/profile-update.yaml create mode 100644 nscharm_policy_vnf/charms/vnf-policy/wheelhouse.txt create mode 100644 nscharm_user_vnf/charms/vnf-user/.travis/profile-update.yaml create mode 100644 nscharm_user_vnf/charms/vnf-user/wheelhouse.txt diff --git a/nscharm_ns/charms/ns/.build.manifest b/nscharm_ns/charms/ns/.build.manifest index c37dc030..427002c4 100644 --- a/nscharm_ns/charms/ns/.build.manifest +++ b/nscharm_ns/charms/ns/.build.manifest @@ -5,15 +5,15 @@ "url": "layer:options" }, { - "rev": "1d2489bff56daf2f8d1c06ee9248bb0094e4ad49", + "rev": "623e69c7b432456fd4364f6e1835424fd6b5425e", "url": "layer:basic" }, { - "rev": "4f3159205af1b3e7a3a532dedcf6b887d2e6d78a", + "rev": "d59fe211bb4c2834352dc8bf62b6d6891e81834c", "url": "layer:osm-ns" }, { - "rev": "aa34ff8bbe4d847102174d9038068af0f6ebe575", + "rev": "b189ed8df5e652bdd886431f3c0220f7f5af87b4", "url": "ns" } ], @@ -28,6 +28,11 @@ "static", "0da5c4dcda27cd6406e5bb81cbf68ddccaf728ac764ec15053a165c1449d87d9" ], + ".travis/profile-update.yaml": [ + "layer:basic", + "static", + "731e20aa59bf61c024d317ad630e478301a9386ccc0afe56e6c1c09db07ac83b" + ], "LICENSE": [ "layer:basic", "static", @@ -56,7 +61,7 @@ "bin/charm-env": [ "layer:basic", "static", - "458c53532c19ee357cbf4209ccc7d811810718ba0ea3b0588b3d3ef040a44b8e" + "fb6a20fac4102a6a4b6ffe903fcf666998f9a95a3647e6f9af7a1eeb44e58fd5" ], "bin/layer_option": [ "layer:options", @@ -141,7 +146,7 @@ "layer.yaml": [ "ns", "dynamic", - "6e5423e1b33993267456ad163ecf0fcd9e921f99214fbc6f23a1d7902468df8f" + "431a730b6fb7851377a021b85ba3ebdfca416448af0bb0618d953928089b1df0" ], "lib/charms/layer/__init__.py": [ "layer:basic", @@ -151,7 +156,7 @@ "lib/charms/layer/basic.py": [ "layer:basic", "static", - "445652dbaa1f0b84a7215da185bcbdff097bb9bbbce11b4c2dbbff90f77719a9" + "3126b5754ad39402ee27e64527044ddd231ed1cd137fcedaffb51e63a635f108" ], "lib/charms/layer/execd.py": [ "layer:basic", @@ -181,7 +186,7 @@ "reactive/ns.py": [ "ns", "static", - "1d1c34c9cdb049552d0c79219cade41e70ae4d76782a305be0e8d4d6928afeb4" + "eea5f82e9332f2129f63c8ff528abf3f556c1c68e25fb08fb9cc2255e0ebdcae" ], "reactive/osm_ns.py": [ "layer:osm-ns", @@ -191,7 +196,7 @@ "requirements.txt": [ "layer:basic", "static", - "0f1c70d27e26005a96d66ad54482877ae20f7737693c833e29dd72bd6ac24892" + "a00f75d80849e5b4fc5ad2e7536f947c25b1a4044b341caa8ee87a92d3a4c804" ], "tests/00-setup": [ "ns", @@ -206,16 +211,21 @@ "version": [ "ns", "dynamic", - "260a9a4811db0cebf6acd4bb4910f460278b8c449d7437a7c908ce864fc6a184" + "a6dd063a3024e9bf54901a897cd7926d5bceab359e6cb0eb9deb34d2eab29367" ], - "wheelhouse/Jinja2-2.10.3.tar.gz": [ + "wheelhouse.txt": [ "layer:basic", "dynamic", - "9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + "7cf3f983dc8f85b0c0ca6d69accdb4f4af842a911625286df09005ed1897d797" ], - "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ + "wheelhouse/Jinja2-2.10.1.tar.gz": [ "layer:basic", "dynamic", + "065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013" + ], + "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ + "__pip__", + "dynamic", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b" ], "wheelhouse/PyYAML-5.2.tar.gz": [ @@ -224,34 +234,39 @@ "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" ], "wheelhouse/Tempita-0.5.2.tar.gz": [ - "layer:basic", + "__pip__", "dynamic", "cacecf0baa674d356641f1d406b8bff1d756d739c46b869a54de515d08e6fc9c" ], - "wheelhouse/charmhelpers-0.20.7.tar.gz": [ + "wheelhouse/charmhelpers-0.20.19.tar.gz": [ "layer:basic", "dynamic", - "e0f8d005d39cded1b0c5997d8ef1d90832341c67ebeb4a334ad1eb348fbd803a" + "74b2d95ec305e5799f0f7d068b0c6e010612ec877feadf32fc51e6bed1f68234" ], - "wheelhouse/charms.reactive-1.3.0.tar.gz": [ + "wheelhouse/charms.reactive-1.3.2.tar.gz": [ "layer:basic", "dynamic", - "82d2c614c82d64bf56e913990f22663e5de64b99db15838abfd9a064f1cc2f51" + "f4633eb37143bf9cc2c0e760a67314954c31fa68ec36d5399d386839cb5d54d5" ], "wheelhouse/netaddr-0.7.19.tar.gz": [ "layer:basic", "dynamic", "38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd" ], + "wheelhouse/pbr-5.5.1.tar.gz": [ + "__pip__", + "dynamic", + "5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9" + ], "wheelhouse/pip-18.1.tar.gz": [ "layer:basic", "dynamic", "c0a292bd977ef590379a3f05d7b7f65135487b67470f6281289a94e015650ea1" ], - "wheelhouse/pyaml-19.12.0.tar.gz": [ - "layer:basic", + "wheelhouse/pyaml-20.4.0.tar.gz": [ + "__pip__", "dynamic", - "b3f636b467864319d7ded1558f86bb305b8612a274f5d443a62dc5eceb1b7176" + "29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71" ], "wheelhouse/setuptools-41.6.0.zip": [ "layer:basic", @@ -263,10 +278,10 @@ "dynamic", "70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a" ], - "wheelhouse/six-1.13.0.tar.gz": [ - "layer:basic", + "wheelhouse/six-1.15.0.tar.gz": [ + "__pip__", "dynamic", - "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259" ], "wheelhouse/wheel-0.33.6.tar.gz": [ "layer:basic", diff --git a/nscharm_ns/charms/ns/.travis/profile-update.yaml b/nscharm_ns/charms/ns/.travis/profile-update.yaml new file mode 100644 index 00000000..57f96eb6 --- /dev/null +++ b/nscharm_ns/charms/ns/.travis/profile-update.yaml @@ -0,0 +1,12 @@ +config: {} +description: Default LXD profile - updated +devices: + eth0: + name: eth0 + parent: lxdbr0 + nictype: bridged + type: nic + root: + path: / + pool: default + type: disk diff --git a/nscharm_ns/charms/ns/bin/charm-env b/nscharm_ns/charms/ns/bin/charm-env index aca7de3c..d211ce9b 100755 --- a/nscharm_ns/charms/ns/bin/charm-env +++ b/nscharm_ns/charms/ns/bin/charm-env @@ -25,7 +25,7 @@ find_charm_dirs() { found_charm_dir="" if [[ -n "$desired_charm" ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - charm_name="$(JUJU_CHARM_DIR="$charm_dir" charm-env python3 -c 'from charmhelpers.core.hookenv import charm_name; print(charm_name())')" + charm_name="$(grep -o '^['\''"]\?name['\''"]\?:.*' $charm_dir/metadata.yaml 2> /dev/null | sed -e 's/.*: *//' -e 's/['\''"]//g')" if [[ "$charm_name" == "$desired_charm" ]]; then if [[ -n "$found_charm_dir" ]]; then >&2 echo "Ambiguous possibilities for JUJU_CHARM_DIR matching '$desired_charm'; please run within a Juju hook context" @@ -43,13 +43,13 @@ find_charm_dirs() { return fi # shellcheck disable=SC2126 - non_subordinates="$(grep -L 'subordinate:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" + non_subordinates="$(grep -L 'subordinate"\?:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" if [[ "$non_subordinates" -gt 1 ]]; then >&2 echo 'Ambiguous possibilities for JUJU_CHARM_DIR; please use --charm or run within a Juju hook context' exit 1 elif [[ "$non_subordinates" -eq 1 ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - if grep -q 'subordinate:.*true' "$charm_dir/metadata.yaml"; then + if grep -q 'subordinate"\?:.*true' "$charm_dir/metadata.yaml"; then continue fi export JUJU_CHARM_DIR="$charm_dir" diff --git a/nscharm_ns/charms/ns/layer.yaml b/nscharm_ns/charms/ns/layer.yaml index efb97d72..8c27bd4d 100644 --- a/nscharm_ns/charms/ns/layer.yaml +++ b/nscharm_ns/charms/ns/layer.yaml @@ -2,7 +2,7 @@ - "layer:options" - "layer:basic" - "layer:osm-ns" -"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt"] +"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt", "unit_tests"] "options": "basic": "use_venv": !!bool "false" diff --git a/nscharm_ns/charms/ns/lib/charms/layer/basic.py b/nscharm_ns/charms/ns/lib/charms/layer/basic.py index 39319591..75072039 100644 --- a/nscharm_ns/charms/ns/lib/charms/layer/basic.py +++ b/nscharm_ns/charms/ns/lib/charms/layer/basic.py @@ -1,6 +1,9 @@ import os import sys +import re import shutil +from distutils.version import LooseVersion +from pkg_resources import Requirement from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep @@ -9,14 +12,55 @@ from charms import layer from charms.layer.execd import execd_preinstall -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d +def _get_subprocess_env(): + env = os.environ.copy() + env['LANG'] = env.get('LANG', 'C.UTF-8') + return env + + +def get_series(): + """ + Return series for a few known OS:es. + Tested as of 2019 november: + * centos6, centos7, rhel6. + * bionic + """ + series = "" + + # Looking for content in /etc/os-release + # works for ubuntu + some centos + if os.path.isfile('/etc/os-release'): + d = {} + with open('/etc/os-release', 'r') as rel: + for l in rel: + if not re.match(r'^\s*$', l): + k, v = l.split('=') + d[k.strip()] = v.strip().replace('"', '') + series = "{ID}{VERSION_ID}".format(**d) + + # Looking for content in /etc/redhat-release + # works for redhat enterprise systems + elif os.path.isfile('/etc/redhat-release'): + with open('/etc/redhat-release', 'r') as redhatlsb: + # CentOS Linux release 7.7.1908 (Core) + line = redhatlsb.readline() + release = int(line.split("release")[1].split()[0][0]) + series = "centos" + str(release) + + # Looking for content in /etc/lsb-release + # works for ubuntu + elif os.path.isfile('/etc/lsb-release'): + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + series = d['DISTRIB_CODENAME'] + + # This is what happens if we cant figure out the OS. + else: + series = "unknown" + return series def bootstrap_charm_deps(): @@ -30,6 +74,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts + + series = get_series() + + # OMG?! is build-essentials needed? + ubuntu_packages = ['python3-pip', + 'python3-setuptools', + 'python3-yaml', + 'python3-dev', + 'python3-wheel', + 'build-essential'] + + # I'm not going to "yum group info "Development Tools" + # omitting above madness + centos_packages = ['python3-pip', + 'python3-setuptools', + 'python3-devel', + 'python3-wheel'] + + packages_needed = [] + if 'centos' in series: + packages_needed = centos_packages + else: + packages_needed = ubuntu_packages + charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') @@ -40,8 +108,9 @@ def bootstrap_charm_deps(): is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' - post_upgrade = os.path.exists('wheelhouse/.upgrade') - is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) + is_post_upgrade = os.path.exists('wheelhouse/.upgraded') + is_upgrade = (not is_post_upgrade and + (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary @@ -50,51 +119,69 @@ def bootstrap_charm_deps(): # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it - if post_upgrade: - os.unlink('wheelhouse/.upgrade') + if is_post_upgrade: + os.unlink('wheelhouse/.upgraded') return - if is_series_upgrade and os.path.exists(venv): - # series upgrade should do a full clear of the venv, rather than just - # updating it, to bring in updates to Python itself - shutil.rmtree(venv) + if os.path.exists(venv): + try: + # focal installs or upgrades prior to PR 160 could leave the venv + # in a broken state which would prevent subsequent charm upgrades + _load_installed_versions(vpip) + except CalledProcessError: + is_broken_venv = True + else: + is_broken_venv = False + if is_upgrade or is_broken_venv: + # All upgrades should do a full clear of the venv, rather than + # just updating it, to bring in updates to Python itself + shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') - open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): + pre_eoan = series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty', + 'ubuntu16.04', 'xenial', + 'ubuntu18.04', 'bionic') + pydistutils_lines = [ + "[easy_install]\n", + "find_links = file://{}/wheelhouse/\n".format(charm_dir), + "no_index=True\n", + "index_url=\n", # deliberately nothing here; disables it. + ] + if pre_eoan: + pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) - fp.writelines([ - "[easy_install]\n", - "allow_hosts = ''\n", - "find_links = file://{}/wheelhouse/\n".format(charm_dir), - ]) - apt_install([ - 'python3-pip', - 'python3-setuptools', - 'python3-yaml', - 'python3-dev', - 'python3-wheel', - 'build-essential', - ]) + fp.writelines(pydistutils_lines) + if 'centos' in series: + yum_install(packages_needed) + else: + apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml - apt_install(cfg.get('packages', [])) + if 'centos' in series: + yum_install(cfg.get('packages', [])) + else: + apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): - series = lsb_release()['DISTRIB_CODENAME'] - if series in ('precise', 'trusty'): + series = get_series() + if series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) + elif 'centos' in series: + yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') - check_call(cmd) + check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: @@ -103,23 +190,36 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') - # need newer pip, to fix spurious Double Requirement error: - # https://github.com/pypa/pip/issues/56 - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'pip']) - # per https://github.com/juju-solutions/layer-basic/issues/110 - # this replaces the setuptools that was copied over from the system on - # venv create with latest setuptools and adds setuptools_scm - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'setuptools', 'setuptools-scm']) - # install the rest of the wheelhouse deps - check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', - '-f', 'wheelhouse'] + glob('wheelhouse/*')) + pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] + # we bundle these packages to work around bugs in older versions (such + # as https://github.com/pypa/pip/issues/56), but if the system already + # provided a newer version, downgrading it can cause other problems + _update_if_newer(pip, pre_install_pkgs) + # install the rest of the wheelhouse deps (extract the pkg names into + # a set so that we can ignore the pre-install packages and let pip + # choose the best version in case there are multiple from layer + # conflicts) + pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) + reinstall_flag = '--force-reinstall' + if not cfg.get('use_venv', True) and pre_eoan: + reinstall_flag = '--ignore-installed' + check_call([pip, 'install', '-U', reinstall_flag, '--no-index', + '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), + env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') + + # install pyyaml for centos7, since, unlike the ubuntu image, the + # default image for centos doesn't include pyyaml; see the discussion: + # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin + if 'centos' in series: + check_call([pip, 'install', '-U', 'pyyaml'], + env=_get_subprocess_env()) + # install python packages from layer options if cfg.get('python_packages'): - check_call([pip, 'install', '-U'] + cfg.get('python_packages')) + check_call([pip, 'install', '-U'] + cfg.get('python_packages'), + env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it @@ -144,6 +244,9 @@ def bootstrap_charm_deps(): os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() + if is_upgrade: + # flag us as having already upgraded so we don't do it again + open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available @@ -151,6 +254,39 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) +def _load_installed_versions(pip): + pip_freeze = check_output([pip, 'freeze']).decode('utf8') + versions = {} + for pkg_ver in pip_freeze.splitlines(): + try: + req = Requirement.parse(pkg_ver) + except ValueError: + continue + versions.update({ + req.project_name: LooseVersion(ver) + for op, ver in req.specs if op == '==' + }) + return versions + + +def _load_wheelhouse_versions(): + versions = {} + for wheel in glob('wheelhouse/*'): + pkg, ver = os.path.basename(wheel).rsplit('-', 1) + # nb: LooseVersion ignores the file extension + versions[pkg.replace('_', '-')] = LooseVersion(ver) + return versions + + +def _update_if_newer(pip, pkgs): + installed = _load_installed_versions(pip) + wheelhouse = _load_wheelhouse_versions() + for pkg in pkgs: + if pkg not in installed or wheelhouse[pkg] > installed[pkg]: + check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', + pkg], env=_get_subprocess_env()) + + def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: @@ -225,7 +361,7 @@ def apt_install(packages): if isinstance(packages, (str, bytes)): packages = [packages] - env = os.environ.copy() + env = _get_subprocess_env() if 'DEBIAN_FRONTEND' not in env: env['DEBIAN_FRONTEND'] = 'noninteractive' @@ -242,7 +378,7 @@ def apt_install(packages): raise try: # sometimes apt-get update needs to be run - check_call(['apt-get', 'update']) + check_call(['apt-get', 'update'], env=env) except CalledProcessError: # sometimes it's a dpkg lock issue pass @@ -251,6 +387,30 @@ def apt_install(packages): break +def yum_install(packages): + """ Installs packages with yum. + This function largely mimics the apt_install function for consistency. + """ + if packages: + env = os.environ.copy() + cmd = ['yum', '-y', 'install'] + for attempt in range(3): + try: + check_call(cmd + packages, env=env) + except CalledProcessError: + if attempt == 2: + raise + try: + check_call(['yum', 'update'], env=env) + except CalledProcessError: + pass + sleep(5) + else: + break + else: + pass + + def init_config_states(): import yaml from charmhelpers.core import hookenv diff --git a/nscharm_ns/charms/ns/reactive/ns.py b/nscharm_ns/charms/ns/reactive/ns.py index 4f13011a..6b583347 100644 --- a/nscharm_ns/charms/ns/reactive/ns.py +++ b/nscharm_ns/charms/ns/reactive/ns.py @@ -1,3 +1,4 @@ +"""asdfself""" from charmhelpers.core.hookenv import ( action_get, action_fail, @@ -17,6 +18,7 @@ import charms.osm.ns import json import traceback + @when_not('ns.installed') def install_ns(): set_flag('ns.installed') @@ -70,7 +72,7 @@ def add_user(client, username, tariff): cfg = config() ns_config_info = json.loads(cfg['ns_config_info']) - log(ns_config_info) + application = ns_config_info['osm-config-mapping']['{}.{}.{}'.format( '1', # member-vnf-index 'userVM', # vdu_id @@ -104,6 +106,7 @@ def set_policy(client, user_id, bw, qos): cfg = config() ns_config_info = json.loads(cfg['ns_config_info']) + application = ns_config_info['osm-config-mapping']['{}.{}.{}'.format( '2', # member-vnf-index 'policyVM', # vdu_id diff --git a/nscharm_ns/charms/ns/requirements.txt b/nscharm_ns/charms/ns/requirements.txt index 28ecacab..55543d9b 100644 --- a/nscharm_ns/charms/ns/requirements.txt +++ b/nscharm_ns/charms/ns/requirements.txt @@ -1,2 +1,3 @@ +mock flake8 pytest diff --git a/nscharm_ns/charms/ns/version b/nscharm_ns/charms/ns/version index 1bfc8f6a..890fe8c2 100644 --- a/nscharm_ns/charms/ns/version +++ b/nscharm_ns/charms/ns/version @@ -1 +1 @@ -aa34ff8-dirty \ No newline at end of file +b189ed8 \ No newline at end of file diff --git a/nscharm_ns/charms/ns/wheelhouse.txt b/nscharm_ns/charms/ns/wheelhouse.txt new file mode 100644 index 00000000..c2337ba8 --- /dev/null +++ b/nscharm_ns/charms/ns/wheelhouse.txt @@ -0,0 +1,16 @@ +# layer:basic +# pip is pinned to <19.0 to avoid https://github.com/pypa/pip/issues/6164 +# even with installing setuptools before upgrading pip ends up with pip seeing +# the older setuptools at the system level if include_system_packages is true +pip>=18.1,<19.0 +# pin Jinja2 and PyYAML to the last versions supporting python 3.4 for trusty +Jinja2<=2.10.1 +PyYAML<=5.2 +setuptools<42 +setuptools-scm<=1.17.0 +charmhelpers>=0.4.0,<1.0.0 +charms.reactive>=0.1.0,<2.0.0 +wheel<0.34 +# pin netaddr to avoid pulling importlib-resources +netaddr<=0.7.19 + diff --git a/nscharm_ns/nscharm_nsd.yaml b/nscharm_ns/nscharm_nsd.yaml index 6af8d866..c8ccb2d1 100644 --- a/nscharm_ns/nscharm_nsd.yaml +++ b/nscharm_ns/nscharm_nsd.yaml @@ -24,7 +24,7 @@ nsd: virtual-link-desc: - id: mgmtnet mgmt-network: 'true' - vim-network-name: mgmt + vim-network-name: osm-ext vnfd-id: - nscharm-user-vnf - nscharm-policy-vnf @@ -39,7 +39,7 @@ nsd: - name: juju-username value: 'admin' - name: juju-password - value: 'd55ce8ab4efa59e7f1b865bce53f55ed' + value: 'a6ffabed913f9d33e1d3201f6f9561b4' - seq: '2' name: add-user parameter: diff --git a/nscharm_policy_vnf/charms/vnf-policy/.build.manifest b/nscharm_policy_vnf/charms/vnf-policy/.build.manifest index 56e04635..eb419db2 100644 --- a/nscharm_policy_vnf/charms/vnf-policy/.build.manifest +++ b/nscharm_policy_vnf/charms/vnf-policy/.build.manifest @@ -5,11 +5,11 @@ "url": "layer:options" }, { - "rev": "1d2489bff56daf2f8d1c06ee9248bb0094e4ad49", + "rev": "623e69c7b432456fd4364f6e1835424fd6b5425e", "url": "layer:basic" }, { - "rev": "29d9dd2e642048f84e1ac388e03756c6cdec3551", + "rev": "009e8631e4d173a71ac53032bbad465a89c6c365", "url": "layer:sshproxy" }, { @@ -32,6 +32,11 @@ "static", "17526a7f7312e7eefb932d1c514b7bc8425fab5bd1ade149e106ecf8bff67358" ], + ".travis/profile-update.yaml": [ + "layer:basic", + "static", + "731e20aa59bf61c024d317ad630e478301a9386ccc0afe56e6c1c09db07ac83b" + ], "LICENSE": [ "layer:basic", "static", @@ -105,7 +110,7 @@ "bin/charm-env": [ "layer:basic", "static", - "458c53532c19ee357cbf4209ccc7d811810718ba0ea3b0588b3d3ef040a44b8e" + "fb6a20fac4102a6a4b6ffe903fcf666998f9a95a3647e6f9af7a1eeb44e58fd5" ], "bin/layer_option": [ "layer:options", @@ -195,7 +200,7 @@ "layer.yaml": [ "vnf-policy", "dynamic", - "44d108f7fb1b6469fd2629c27c551f5d961e94e8285f785324b96b9c36c372d9" + "309a53cb906038c4732b442265d7f1f3ca662a38660763721f2a0a1f11fc455a" ], "lib/charms/layer/__init__.py": [ "layer:basic", @@ -205,7 +210,7 @@ "lib/charms/layer/basic.py": [ "layer:basic", "static", - "445652dbaa1f0b84a7215da185bcbdff097bb9bbbce11b4c2dbbff90f77719a9" + "3126b5754ad39402ee27e64527044ddd231ed1cd137fcedaffb51e63a635f108" ], "lib/charms/layer/execd.py": [ "layer:basic", @@ -250,7 +255,7 @@ "requirements.txt": [ "layer:basic", "static", - "0f1c70d27e26005a96d66ad54482877ae20f7737693c833e29dd72bd6ac24892" + "a00f75d80849e5b4fc5ad2e7536f947c25b1a4044b341caa8ee87a92d3a4c804" ], "tests/00-setup": [ "vnf-policy", @@ -267,13 +272,18 @@ "dynamic", "ddf4577d49e7c71e11e6dd4f85ad6e74551d5ba89f52b6370de9c141680f9c7f" ], - "wheelhouse/Jinja2-2.10.3.tar.gz": [ + "wheelhouse.txt": [ + "layer:sshproxy", + "dynamic", + "6ad4334a685c25c67bf07a9271a6d62afb85e3642fb5bc16e71852fba412ef8e" + ], + "wheelhouse/Jinja2-2.10.1.tar.gz": [ "layer:basic", "dynamic", - "9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + "065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ - "layer:basic", + "__pip__", "dynamic", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b" ], @@ -283,24 +293,24 @@ "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" ], "wheelhouse/Tempita-0.5.2.tar.gz": [ - "layer:basic", + "__pip__", "dynamic", "cacecf0baa674d356641f1d406b8bff1d756d739c46b869a54de515d08e6fc9c" ], - "wheelhouse/charmhelpers-0.20.7.tar.gz": [ + "wheelhouse/charmhelpers-0.20.19.tar.gz": [ "layer:basic", "dynamic", - "e0f8d005d39cded1b0c5997d8ef1d90832341c67ebeb4a334ad1eb348fbd803a" + "74b2d95ec305e5799f0f7d068b0c6e010612ec877feadf32fc51e6bed1f68234" ], - "wheelhouse/charms.reactive-1.3.0.tar.gz": [ + "wheelhouse/charms.reactive-1.3.2.tar.gz": [ "layer:basic", "dynamic", - "82d2c614c82d64bf56e913990f22663e5de64b99db15838abfd9a064f1cc2f51" + "f4633eb37143bf9cc2c0e760a67314954c31fa68ec36d5399d386839cb5d54d5" ], - "wheelhouse/ecdsa-0.14.1.tar.gz": [ - "layer:sshproxy", + "wheelhouse/ecdsa-0.16.1.tar.gz": [ + "__pip__", "dynamic", - "64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e" + "cfc046a2ddd425adbd1a78b3c46f0d1325c657811c0f45ecc3a0a6236c1e50ff" ], "wheelhouse/netaddr-0.7.19.tar.gz": [ "layer:basic", @@ -312,18 +322,23 @@ "dynamic", "97d932fdb4fec9aadf6bea368123f3ee15b92199f92eb62666370c7fed62d072" ], + "wheelhouse/pbr-5.5.1.tar.gz": [ + "__pip__", + "dynamic", + "5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9" + ], "wheelhouse/pip-18.1.tar.gz": [ "layer:basic", "dynamic", "c0a292bd977ef590379a3f05d7b7f65135487b67470f6281289a94e015650ea1" ], - "wheelhouse/pyaml-19.4.1.tar.gz": [ - "layer:basic", + "wheelhouse/pyaml-20.4.0.tar.gz": [ + "__pip__", "dynamic", - "c79ae98ececda136a034115ca178ee8bf3aa7df236c488c2f55d12f177b88f1e" + "29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71" ], "wheelhouse/pycrypto-2.6.1.tar.gz": [ - "layer:sshproxy", + "__pip__", "dynamic", "f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c" ], @@ -337,10 +352,10 @@ "dynamic", "70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a" ], - "wheelhouse/six-1.13.0.tar.gz": [ - "layer:sshproxy", + "wheelhouse/six-1.15.0.tar.gz": [ + "__pip__", "dynamic", - "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259" ], "wheelhouse/wheel-0.33.6.tar.gz": [ "layer:basic", diff --git a/nscharm_policy_vnf/charms/vnf-policy/.travis/profile-update.yaml b/nscharm_policy_vnf/charms/vnf-policy/.travis/profile-update.yaml new file mode 100644 index 00000000..57f96eb6 --- /dev/null +++ b/nscharm_policy_vnf/charms/vnf-policy/.travis/profile-update.yaml @@ -0,0 +1,12 @@ +config: {} +description: Default LXD profile - updated +devices: + eth0: + name: eth0 + parent: lxdbr0 + nictype: bridged + type: nic + root: + path: / + pool: default + type: disk diff --git a/nscharm_policy_vnf/charms/vnf-policy/bin/charm-env b/nscharm_policy_vnf/charms/vnf-policy/bin/charm-env index aca7de3c..d211ce9b 100755 --- a/nscharm_policy_vnf/charms/vnf-policy/bin/charm-env +++ b/nscharm_policy_vnf/charms/vnf-policy/bin/charm-env @@ -25,7 +25,7 @@ find_charm_dirs() { found_charm_dir="" if [[ -n "$desired_charm" ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - charm_name="$(JUJU_CHARM_DIR="$charm_dir" charm-env python3 -c 'from charmhelpers.core.hookenv import charm_name; print(charm_name())')" + charm_name="$(grep -o '^['\''"]\?name['\''"]\?:.*' $charm_dir/metadata.yaml 2> /dev/null | sed -e 's/.*: *//' -e 's/['\''"]//g')" if [[ "$charm_name" == "$desired_charm" ]]; then if [[ -n "$found_charm_dir" ]]; then >&2 echo "Ambiguous possibilities for JUJU_CHARM_DIR matching '$desired_charm'; please run within a Juju hook context" @@ -43,13 +43,13 @@ find_charm_dirs() { return fi # shellcheck disable=SC2126 - non_subordinates="$(grep -L 'subordinate:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" + non_subordinates="$(grep -L 'subordinate"\?:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" if [[ "$non_subordinates" -gt 1 ]]; then >&2 echo 'Ambiguous possibilities for JUJU_CHARM_DIR; please use --charm or run within a Juju hook context' exit 1 elif [[ "$non_subordinates" -eq 1 ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - if grep -q 'subordinate:.*true' "$charm_dir/metadata.yaml"; then + if grep -q 'subordinate"\?:.*true' "$charm_dir/metadata.yaml"; then continue fi export JUJU_CHARM_DIR="$charm_dir" diff --git a/nscharm_policy_vnf/charms/vnf-policy/layer.yaml b/nscharm_policy_vnf/charms/vnf-policy/layer.yaml index baddbc23..98e6ce19 100644 --- a/nscharm_policy_vnf/charms/vnf-policy/layer.yaml +++ b/nscharm_policy_vnf/charms/vnf-policy/layer.yaml @@ -3,7 +3,7 @@ - "layer:basic" - "layer:sshproxy" - "layer:vnfproxy" -"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt"] +"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt", "unit_tests"] "options": "basic": "use_venv": !!bool "false" diff --git a/nscharm_policy_vnf/charms/vnf-policy/lib/charms/layer/basic.py b/nscharm_policy_vnf/charms/vnf-policy/lib/charms/layer/basic.py index 39319591..75072039 100644 --- a/nscharm_policy_vnf/charms/vnf-policy/lib/charms/layer/basic.py +++ b/nscharm_policy_vnf/charms/vnf-policy/lib/charms/layer/basic.py @@ -1,6 +1,9 @@ import os import sys +import re import shutil +from distutils.version import LooseVersion +from pkg_resources import Requirement from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep @@ -9,14 +12,55 @@ from charms import layer from charms.layer.execd import execd_preinstall -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d +def _get_subprocess_env(): + env = os.environ.copy() + env['LANG'] = env.get('LANG', 'C.UTF-8') + return env + + +def get_series(): + """ + Return series for a few known OS:es. + Tested as of 2019 november: + * centos6, centos7, rhel6. + * bionic + """ + series = "" + + # Looking for content in /etc/os-release + # works for ubuntu + some centos + if os.path.isfile('/etc/os-release'): + d = {} + with open('/etc/os-release', 'r') as rel: + for l in rel: + if not re.match(r'^\s*$', l): + k, v = l.split('=') + d[k.strip()] = v.strip().replace('"', '') + series = "{ID}{VERSION_ID}".format(**d) + + # Looking for content in /etc/redhat-release + # works for redhat enterprise systems + elif os.path.isfile('/etc/redhat-release'): + with open('/etc/redhat-release', 'r') as redhatlsb: + # CentOS Linux release 7.7.1908 (Core) + line = redhatlsb.readline() + release = int(line.split("release")[1].split()[0][0]) + series = "centos" + str(release) + + # Looking for content in /etc/lsb-release + # works for ubuntu + elif os.path.isfile('/etc/lsb-release'): + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + series = d['DISTRIB_CODENAME'] + + # This is what happens if we cant figure out the OS. + else: + series = "unknown" + return series def bootstrap_charm_deps(): @@ -30,6 +74,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts + + series = get_series() + + # OMG?! is build-essentials needed? + ubuntu_packages = ['python3-pip', + 'python3-setuptools', + 'python3-yaml', + 'python3-dev', + 'python3-wheel', + 'build-essential'] + + # I'm not going to "yum group info "Development Tools" + # omitting above madness + centos_packages = ['python3-pip', + 'python3-setuptools', + 'python3-devel', + 'python3-wheel'] + + packages_needed = [] + if 'centos' in series: + packages_needed = centos_packages + else: + packages_needed = ubuntu_packages + charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') @@ -40,8 +108,9 @@ def bootstrap_charm_deps(): is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' - post_upgrade = os.path.exists('wheelhouse/.upgrade') - is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) + is_post_upgrade = os.path.exists('wheelhouse/.upgraded') + is_upgrade = (not is_post_upgrade and + (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary @@ -50,51 +119,69 @@ def bootstrap_charm_deps(): # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it - if post_upgrade: - os.unlink('wheelhouse/.upgrade') + if is_post_upgrade: + os.unlink('wheelhouse/.upgraded') return - if is_series_upgrade and os.path.exists(venv): - # series upgrade should do a full clear of the venv, rather than just - # updating it, to bring in updates to Python itself - shutil.rmtree(venv) + if os.path.exists(venv): + try: + # focal installs or upgrades prior to PR 160 could leave the venv + # in a broken state which would prevent subsequent charm upgrades + _load_installed_versions(vpip) + except CalledProcessError: + is_broken_venv = True + else: + is_broken_venv = False + if is_upgrade or is_broken_venv: + # All upgrades should do a full clear of the venv, rather than + # just updating it, to bring in updates to Python itself + shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') - open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): + pre_eoan = series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty', + 'ubuntu16.04', 'xenial', + 'ubuntu18.04', 'bionic') + pydistutils_lines = [ + "[easy_install]\n", + "find_links = file://{}/wheelhouse/\n".format(charm_dir), + "no_index=True\n", + "index_url=\n", # deliberately nothing here; disables it. + ] + if pre_eoan: + pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) - fp.writelines([ - "[easy_install]\n", - "allow_hosts = ''\n", - "find_links = file://{}/wheelhouse/\n".format(charm_dir), - ]) - apt_install([ - 'python3-pip', - 'python3-setuptools', - 'python3-yaml', - 'python3-dev', - 'python3-wheel', - 'build-essential', - ]) + fp.writelines(pydistutils_lines) + if 'centos' in series: + yum_install(packages_needed) + else: + apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml - apt_install(cfg.get('packages', [])) + if 'centos' in series: + yum_install(cfg.get('packages', [])) + else: + apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): - series = lsb_release()['DISTRIB_CODENAME'] - if series in ('precise', 'trusty'): + series = get_series() + if series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) + elif 'centos' in series: + yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') - check_call(cmd) + check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: @@ -103,23 +190,36 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') - # need newer pip, to fix spurious Double Requirement error: - # https://github.com/pypa/pip/issues/56 - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'pip']) - # per https://github.com/juju-solutions/layer-basic/issues/110 - # this replaces the setuptools that was copied over from the system on - # venv create with latest setuptools and adds setuptools_scm - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'setuptools', 'setuptools-scm']) - # install the rest of the wheelhouse deps - check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', - '-f', 'wheelhouse'] + glob('wheelhouse/*')) + pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] + # we bundle these packages to work around bugs in older versions (such + # as https://github.com/pypa/pip/issues/56), but if the system already + # provided a newer version, downgrading it can cause other problems + _update_if_newer(pip, pre_install_pkgs) + # install the rest of the wheelhouse deps (extract the pkg names into + # a set so that we can ignore the pre-install packages and let pip + # choose the best version in case there are multiple from layer + # conflicts) + pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) + reinstall_flag = '--force-reinstall' + if not cfg.get('use_venv', True) and pre_eoan: + reinstall_flag = '--ignore-installed' + check_call([pip, 'install', '-U', reinstall_flag, '--no-index', + '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), + env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') + + # install pyyaml for centos7, since, unlike the ubuntu image, the + # default image for centos doesn't include pyyaml; see the discussion: + # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin + if 'centos' in series: + check_call([pip, 'install', '-U', 'pyyaml'], + env=_get_subprocess_env()) + # install python packages from layer options if cfg.get('python_packages'): - check_call([pip, 'install', '-U'] + cfg.get('python_packages')) + check_call([pip, 'install', '-U'] + cfg.get('python_packages'), + env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it @@ -144,6 +244,9 @@ def bootstrap_charm_deps(): os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() + if is_upgrade: + # flag us as having already upgraded so we don't do it again + open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available @@ -151,6 +254,39 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) +def _load_installed_versions(pip): + pip_freeze = check_output([pip, 'freeze']).decode('utf8') + versions = {} + for pkg_ver in pip_freeze.splitlines(): + try: + req = Requirement.parse(pkg_ver) + except ValueError: + continue + versions.update({ + req.project_name: LooseVersion(ver) + for op, ver in req.specs if op == '==' + }) + return versions + + +def _load_wheelhouse_versions(): + versions = {} + for wheel in glob('wheelhouse/*'): + pkg, ver = os.path.basename(wheel).rsplit('-', 1) + # nb: LooseVersion ignores the file extension + versions[pkg.replace('_', '-')] = LooseVersion(ver) + return versions + + +def _update_if_newer(pip, pkgs): + installed = _load_installed_versions(pip) + wheelhouse = _load_wheelhouse_versions() + for pkg in pkgs: + if pkg not in installed or wheelhouse[pkg] > installed[pkg]: + check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', + pkg], env=_get_subprocess_env()) + + def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: @@ -225,7 +361,7 @@ def apt_install(packages): if isinstance(packages, (str, bytes)): packages = [packages] - env = os.environ.copy() + env = _get_subprocess_env() if 'DEBIAN_FRONTEND' not in env: env['DEBIAN_FRONTEND'] = 'noninteractive' @@ -242,7 +378,7 @@ def apt_install(packages): raise try: # sometimes apt-get update needs to be run - check_call(['apt-get', 'update']) + check_call(['apt-get', 'update'], env=env) except CalledProcessError: # sometimes it's a dpkg lock issue pass @@ -251,6 +387,30 @@ def apt_install(packages): break +def yum_install(packages): + """ Installs packages with yum. + This function largely mimics the apt_install function for consistency. + """ + if packages: + env = os.environ.copy() + cmd = ['yum', '-y', 'install'] + for attempt in range(3): + try: + check_call(cmd + packages, env=env) + except CalledProcessError: + if attempt == 2: + raise + try: + check_call(['yum', 'update'], env=env) + except CalledProcessError: + pass + sleep(5) + else: + break + else: + pass + + def init_config_states(): import yaml from charmhelpers.core import hookenv diff --git a/nscharm_policy_vnf/charms/vnf-policy/requirements.txt b/nscharm_policy_vnf/charms/vnf-policy/requirements.txt index 28ecacab..55543d9b 100644 --- a/nscharm_policy_vnf/charms/vnf-policy/requirements.txt +++ b/nscharm_policy_vnf/charms/vnf-policy/requirements.txt @@ -1,2 +1,3 @@ +mock flake8 pytest diff --git a/nscharm_policy_vnf/charms/vnf-policy/wheelhouse.txt b/nscharm_policy_vnf/charms/vnf-policy/wheelhouse.txt new file mode 100644 index 00000000..1648a361 --- /dev/null +++ b/nscharm_policy_vnf/charms/vnf-policy/wheelhouse.txt @@ -0,0 +1,36 @@ +# layer:basic +# pip is pinned to <19.0 to avoid https://github.com/pypa/pip/issues/6164 +# even with installing setuptools before upgrading pip ends up with pip seeing +# the older setuptools at the system level if include_system_packages is true +pip>=18.1,<19.0 +# pin Jinja2 and PyYAML to the last versions supporting python 3.4 for trusty +Jinja2<=2.10.1 +PyYAML<=5.2 +setuptools<42 +setuptools-scm<=1.17.0 +charmhelpers>=0.4.0,<1.0.0 +charms.reactive>=0.1.0,<2.0.0 +wheel<0.34 +# pin netaddr to avoid pulling importlib-resources +netaddr<=0.7.19 + +# layer:sshproxy +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +paramiko>=1.16.0,<1.17 + diff --git a/nscharm_user_vnf/charms/vnf-user/.build.manifest b/nscharm_user_vnf/charms/vnf-user/.build.manifest index be8ebcf8..708c0968 100644 --- a/nscharm_user_vnf/charms/vnf-user/.build.manifest +++ b/nscharm_user_vnf/charms/vnf-user/.build.manifest @@ -5,11 +5,11 @@ "url": "layer:options" }, { - "rev": "1d2489bff56daf2f8d1c06ee9248bb0094e4ad49", + "rev": "623e69c7b432456fd4364f6e1835424fd6b5425e", "url": "layer:basic" }, { - "rev": "29d9dd2e642048f84e1ac388e03756c6cdec3551", + "rev": "009e8631e4d173a71ac53032bbad465a89c6c365", "url": "layer:sshproxy" }, { @@ -32,6 +32,11 @@ "static", "17526a7f7312e7eefb932d1c514b7bc8425fab5bd1ade149e106ecf8bff67358" ], + ".travis/profile-update.yaml": [ + "layer:basic", + "static", + "731e20aa59bf61c024d317ad630e478301a9386ccc0afe56e6c1c09db07ac83b" + ], "LICENSE": [ "layer:basic", "static", @@ -105,7 +110,7 @@ "bin/charm-env": [ "layer:basic", "static", - "458c53532c19ee357cbf4209ccc7d811810718ba0ea3b0588b3d3ef040a44b8e" + "fb6a20fac4102a6a4b6ffe903fcf666998f9a95a3647e6f9af7a1eeb44e58fd5" ], "bin/layer_option": [ "layer:options", @@ -195,7 +200,7 @@ "layer.yaml": [ "vnf-user", "dynamic", - "9082d2e11f485d7af081f99542c949e3580add8dcf00a72c90bd096d850b75e1" + "05c0095f9c08579d4ad3d8a4c600fc296036eaa1b708998bbadb2f02449c5354" ], "lib/charms/layer/__init__.py": [ "layer:basic", @@ -205,7 +210,7 @@ "lib/charms/layer/basic.py": [ "layer:basic", "static", - "445652dbaa1f0b84a7215da185bcbdff097bb9bbbce11b4c2dbbff90f77719a9" + "3126b5754ad39402ee27e64527044ddd231ed1cd137fcedaffb51e63a635f108" ], "lib/charms/layer/execd.py": [ "layer:basic", @@ -250,7 +255,7 @@ "requirements.txt": [ "layer:basic", "static", - "0f1c70d27e26005a96d66ad54482877ae20f7737693c833e29dd72bd6ac24892" + "a00f75d80849e5b4fc5ad2e7536f947c25b1a4044b341caa8ee87a92d3a4c804" ], "tests/00-setup": [ "vnf-user", @@ -267,13 +272,18 @@ "dynamic", "02b6a61e71b793429e0391c6170475f2d35a07035a1f9db392271e8c44e76dd2" ], - "wheelhouse/Jinja2-2.10.3.tar.gz": [ + "wheelhouse.txt": [ + "layer:sshproxy", + "dynamic", + "6ad4334a685c25c67bf07a9271a6d62afb85e3642fb5bc16e71852fba412ef8e" + ], + "wheelhouse/Jinja2-2.10.1.tar.gz": [ "layer:basic", "dynamic", - "9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + "065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ - "layer:basic", + "__pip__", "dynamic", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b" ], @@ -283,24 +293,24 @@ "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" ], "wheelhouse/Tempita-0.5.2.tar.gz": [ - "layer:basic", + "__pip__", "dynamic", "cacecf0baa674d356641f1d406b8bff1d756d739c46b869a54de515d08e6fc9c" ], - "wheelhouse/charmhelpers-0.20.7.tar.gz": [ + "wheelhouse/charmhelpers-0.20.19.tar.gz": [ "layer:basic", "dynamic", - "e0f8d005d39cded1b0c5997d8ef1d90832341c67ebeb4a334ad1eb348fbd803a" + "74b2d95ec305e5799f0f7d068b0c6e010612ec877feadf32fc51e6bed1f68234" ], - "wheelhouse/charms.reactive-1.3.0.tar.gz": [ + "wheelhouse/charms.reactive-1.3.2.tar.gz": [ "layer:basic", "dynamic", - "82d2c614c82d64bf56e913990f22663e5de64b99db15838abfd9a064f1cc2f51" + "f4633eb37143bf9cc2c0e760a67314954c31fa68ec36d5399d386839cb5d54d5" ], - "wheelhouse/ecdsa-0.14.1.tar.gz": [ - "layer:sshproxy", + "wheelhouse/ecdsa-0.16.1.tar.gz": [ + "__pip__", "dynamic", - "64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e" + "cfc046a2ddd425adbd1a78b3c46f0d1325c657811c0f45ecc3a0a6236c1e50ff" ], "wheelhouse/netaddr-0.7.19.tar.gz": [ "layer:basic", @@ -312,18 +322,23 @@ "dynamic", "97d932fdb4fec9aadf6bea368123f3ee15b92199f92eb62666370c7fed62d072" ], + "wheelhouse/pbr-5.5.1.tar.gz": [ + "__pip__", + "dynamic", + "5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9" + ], "wheelhouse/pip-18.1.tar.gz": [ "layer:basic", "dynamic", "c0a292bd977ef590379a3f05d7b7f65135487b67470f6281289a94e015650ea1" ], - "wheelhouse/pyaml-19.4.1.tar.gz": [ - "layer:basic", + "wheelhouse/pyaml-20.4.0.tar.gz": [ + "__pip__", "dynamic", - "c79ae98ececda136a034115ca178ee8bf3aa7df236c488c2f55d12f177b88f1e" + "29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71" ], "wheelhouse/pycrypto-2.6.1.tar.gz": [ - "layer:sshproxy", + "__pip__", "dynamic", "f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c" ], @@ -337,10 +352,10 @@ "dynamic", "70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a" ], - "wheelhouse/six-1.13.0.tar.gz": [ - "layer:sshproxy", + "wheelhouse/six-1.15.0.tar.gz": [ + "__pip__", "dynamic", - "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259" ], "wheelhouse/wheel-0.33.6.tar.gz": [ "layer:basic", diff --git a/nscharm_user_vnf/charms/vnf-user/.travis/profile-update.yaml b/nscharm_user_vnf/charms/vnf-user/.travis/profile-update.yaml new file mode 100644 index 00000000..57f96eb6 --- /dev/null +++ b/nscharm_user_vnf/charms/vnf-user/.travis/profile-update.yaml @@ -0,0 +1,12 @@ +config: {} +description: Default LXD profile - updated +devices: + eth0: + name: eth0 + parent: lxdbr0 + nictype: bridged + type: nic + root: + path: / + pool: default + type: disk diff --git a/nscharm_user_vnf/charms/vnf-user/bin/charm-env b/nscharm_user_vnf/charms/vnf-user/bin/charm-env index aca7de3c..d211ce9b 100755 --- a/nscharm_user_vnf/charms/vnf-user/bin/charm-env +++ b/nscharm_user_vnf/charms/vnf-user/bin/charm-env @@ -25,7 +25,7 @@ find_charm_dirs() { found_charm_dir="" if [[ -n "$desired_charm" ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - charm_name="$(JUJU_CHARM_DIR="$charm_dir" charm-env python3 -c 'from charmhelpers.core.hookenv import charm_name; print(charm_name())')" + charm_name="$(grep -o '^['\''"]\?name['\''"]\?:.*' $charm_dir/metadata.yaml 2> /dev/null | sed -e 's/.*: *//' -e 's/['\''"]//g')" if [[ "$charm_name" == "$desired_charm" ]]; then if [[ -n "$found_charm_dir" ]]; then >&2 echo "Ambiguous possibilities for JUJU_CHARM_DIR matching '$desired_charm'; please run within a Juju hook context" @@ -43,13 +43,13 @@ find_charm_dirs() { return fi # shellcheck disable=SC2126 - non_subordinates="$(grep -L 'subordinate:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" + non_subordinates="$(grep -L 'subordinate"\?:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" if [[ "$non_subordinates" -gt 1 ]]; then >&2 echo 'Ambiguous possibilities for JUJU_CHARM_DIR; please use --charm or run within a Juju hook context' exit 1 elif [[ "$non_subordinates" -eq 1 ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do - if grep -q 'subordinate:.*true' "$charm_dir/metadata.yaml"; then + if grep -q 'subordinate"\?:.*true' "$charm_dir/metadata.yaml"; then continue fi export JUJU_CHARM_DIR="$charm_dir" diff --git a/nscharm_user_vnf/charms/vnf-user/layer.yaml b/nscharm_user_vnf/charms/vnf-user/layer.yaml index dd758c57..d8ab5ff8 100644 --- a/nscharm_user_vnf/charms/vnf-user/layer.yaml +++ b/nscharm_user_vnf/charms/vnf-user/layer.yaml @@ -3,7 +3,7 @@ - "layer:basic" - "layer:sshproxy" - "layer:vnfproxy" -"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt"] +"exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt", "unit_tests"] "options": "basic": "use_venv": !!bool "false" diff --git a/nscharm_user_vnf/charms/vnf-user/lib/charms/layer/basic.py b/nscharm_user_vnf/charms/vnf-user/lib/charms/layer/basic.py index 39319591..75072039 100644 --- a/nscharm_user_vnf/charms/vnf-user/lib/charms/layer/basic.py +++ b/nscharm_user_vnf/charms/vnf-user/lib/charms/layer/basic.py @@ -1,6 +1,9 @@ import os import sys +import re import shutil +from distutils.version import LooseVersion +from pkg_resources import Requirement from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep @@ -9,14 +12,55 @@ from charms import layer from charms.layer.execd import execd_preinstall -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d +def _get_subprocess_env(): + env = os.environ.copy() + env['LANG'] = env.get('LANG', 'C.UTF-8') + return env + + +def get_series(): + """ + Return series for a few known OS:es. + Tested as of 2019 november: + * centos6, centos7, rhel6. + * bionic + """ + series = "" + + # Looking for content in /etc/os-release + # works for ubuntu + some centos + if os.path.isfile('/etc/os-release'): + d = {} + with open('/etc/os-release', 'r') as rel: + for l in rel: + if not re.match(r'^\s*$', l): + k, v = l.split('=') + d[k.strip()] = v.strip().replace('"', '') + series = "{ID}{VERSION_ID}".format(**d) + + # Looking for content in /etc/redhat-release + # works for redhat enterprise systems + elif os.path.isfile('/etc/redhat-release'): + with open('/etc/redhat-release', 'r') as redhatlsb: + # CentOS Linux release 7.7.1908 (Core) + line = redhatlsb.readline() + release = int(line.split("release")[1].split()[0][0]) + series = "centos" + str(release) + + # Looking for content in /etc/lsb-release + # works for ubuntu + elif os.path.isfile('/etc/lsb-release'): + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + series = d['DISTRIB_CODENAME'] + + # This is what happens if we cant figure out the OS. + else: + series = "unknown" + return series def bootstrap_charm_deps(): @@ -30,6 +74,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts + + series = get_series() + + # OMG?! is build-essentials needed? + ubuntu_packages = ['python3-pip', + 'python3-setuptools', + 'python3-yaml', + 'python3-dev', + 'python3-wheel', + 'build-essential'] + + # I'm not going to "yum group info "Development Tools" + # omitting above madness + centos_packages = ['python3-pip', + 'python3-setuptools', + 'python3-devel', + 'python3-wheel'] + + packages_needed = [] + if 'centos' in series: + packages_needed = centos_packages + else: + packages_needed = ubuntu_packages + charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') @@ -40,8 +108,9 @@ def bootstrap_charm_deps(): is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' - post_upgrade = os.path.exists('wheelhouse/.upgrade') - is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) + is_post_upgrade = os.path.exists('wheelhouse/.upgraded') + is_upgrade = (not is_post_upgrade and + (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary @@ -50,51 +119,69 @@ def bootstrap_charm_deps(): # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it - if post_upgrade: - os.unlink('wheelhouse/.upgrade') + if is_post_upgrade: + os.unlink('wheelhouse/.upgraded') return - if is_series_upgrade and os.path.exists(venv): - # series upgrade should do a full clear of the venv, rather than just - # updating it, to bring in updates to Python itself - shutil.rmtree(venv) + if os.path.exists(venv): + try: + # focal installs or upgrades prior to PR 160 could leave the venv + # in a broken state which would prevent subsequent charm upgrades + _load_installed_versions(vpip) + except CalledProcessError: + is_broken_venv = True + else: + is_broken_venv = False + if is_upgrade or is_broken_venv: + # All upgrades should do a full clear of the venv, rather than + # just updating it, to bring in updates to Python itself + shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') - open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): + pre_eoan = series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty', + 'ubuntu16.04', 'xenial', + 'ubuntu18.04', 'bionic') + pydistutils_lines = [ + "[easy_install]\n", + "find_links = file://{}/wheelhouse/\n".format(charm_dir), + "no_index=True\n", + "index_url=\n", # deliberately nothing here; disables it. + ] + if pre_eoan: + pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) - fp.writelines([ - "[easy_install]\n", - "allow_hosts = ''\n", - "find_links = file://{}/wheelhouse/\n".format(charm_dir), - ]) - apt_install([ - 'python3-pip', - 'python3-setuptools', - 'python3-yaml', - 'python3-dev', - 'python3-wheel', - 'build-essential', - ]) + fp.writelines(pydistutils_lines) + if 'centos' in series: + yum_install(packages_needed) + else: + apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml - apt_install(cfg.get('packages', [])) + if 'centos' in series: + yum_install(cfg.get('packages', [])) + else: + apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): - series = lsb_release()['DISTRIB_CODENAME'] - if series in ('precise', 'trusty'): + series = get_series() + if series in ('ubuntu12.04', 'precise', + 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) + elif 'centos' in series: + yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') - check_call(cmd) + check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: @@ -103,23 +190,36 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') - # need newer pip, to fix spurious Double Requirement error: - # https://github.com/pypa/pip/issues/56 - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'pip']) - # per https://github.com/juju-solutions/layer-basic/issues/110 - # this replaces the setuptools that was copied over from the system on - # venv create with latest setuptools and adds setuptools_scm - check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', - 'setuptools', 'setuptools-scm']) - # install the rest of the wheelhouse deps - check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', - '-f', 'wheelhouse'] + glob('wheelhouse/*')) + pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] + # we bundle these packages to work around bugs in older versions (such + # as https://github.com/pypa/pip/issues/56), but if the system already + # provided a newer version, downgrading it can cause other problems + _update_if_newer(pip, pre_install_pkgs) + # install the rest of the wheelhouse deps (extract the pkg names into + # a set so that we can ignore the pre-install packages and let pip + # choose the best version in case there are multiple from layer + # conflicts) + pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) + reinstall_flag = '--force-reinstall' + if not cfg.get('use_venv', True) and pre_eoan: + reinstall_flag = '--ignore-installed' + check_call([pip, 'install', '-U', reinstall_flag, '--no-index', + '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), + env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') + + # install pyyaml for centos7, since, unlike the ubuntu image, the + # default image for centos doesn't include pyyaml; see the discussion: + # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin + if 'centos' in series: + check_call([pip, 'install', '-U', 'pyyaml'], + env=_get_subprocess_env()) + # install python packages from layer options if cfg.get('python_packages'): - check_call([pip, 'install', '-U'] + cfg.get('python_packages')) + check_call([pip, 'install', '-U'] + cfg.get('python_packages'), + env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it @@ -144,6 +244,9 @@ def bootstrap_charm_deps(): os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() + if is_upgrade: + # flag us as having already upgraded so we don't do it again + open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available @@ -151,6 +254,39 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) +def _load_installed_versions(pip): + pip_freeze = check_output([pip, 'freeze']).decode('utf8') + versions = {} + for pkg_ver in pip_freeze.splitlines(): + try: + req = Requirement.parse(pkg_ver) + except ValueError: + continue + versions.update({ + req.project_name: LooseVersion(ver) + for op, ver in req.specs if op == '==' + }) + return versions + + +def _load_wheelhouse_versions(): + versions = {} + for wheel in glob('wheelhouse/*'): + pkg, ver = os.path.basename(wheel).rsplit('-', 1) + # nb: LooseVersion ignores the file extension + versions[pkg.replace('_', '-')] = LooseVersion(ver) + return versions + + +def _update_if_newer(pip, pkgs): + installed = _load_installed_versions(pip) + wheelhouse = _load_wheelhouse_versions() + for pkg in pkgs: + if pkg not in installed or wheelhouse[pkg] > installed[pkg]: + check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', + pkg], env=_get_subprocess_env()) + + def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: @@ -225,7 +361,7 @@ def apt_install(packages): if isinstance(packages, (str, bytes)): packages = [packages] - env = os.environ.copy() + env = _get_subprocess_env() if 'DEBIAN_FRONTEND' not in env: env['DEBIAN_FRONTEND'] = 'noninteractive' @@ -242,7 +378,7 @@ def apt_install(packages): raise try: # sometimes apt-get update needs to be run - check_call(['apt-get', 'update']) + check_call(['apt-get', 'update'], env=env) except CalledProcessError: # sometimes it's a dpkg lock issue pass @@ -251,6 +387,30 @@ def apt_install(packages): break +def yum_install(packages): + """ Installs packages with yum. + This function largely mimics the apt_install function for consistency. + """ + if packages: + env = os.environ.copy() + cmd = ['yum', '-y', 'install'] + for attempt in range(3): + try: + check_call(cmd + packages, env=env) + except CalledProcessError: + if attempt == 2: + raise + try: + check_call(['yum', 'update'], env=env) + except CalledProcessError: + pass + sleep(5) + else: + break + else: + pass + + def init_config_states(): import yaml from charmhelpers.core import hookenv diff --git a/nscharm_user_vnf/charms/vnf-user/requirements.txt b/nscharm_user_vnf/charms/vnf-user/requirements.txt index 28ecacab..55543d9b 100644 --- a/nscharm_user_vnf/charms/vnf-user/requirements.txt +++ b/nscharm_user_vnf/charms/vnf-user/requirements.txt @@ -1,2 +1,3 @@ +mock flake8 pytest diff --git a/nscharm_user_vnf/charms/vnf-user/wheelhouse.txt b/nscharm_user_vnf/charms/vnf-user/wheelhouse.txt new file mode 100644 index 00000000..1648a361 --- /dev/null +++ b/nscharm_user_vnf/charms/vnf-user/wheelhouse.txt @@ -0,0 +1,36 @@ +# layer:basic +# pip is pinned to <19.0 to avoid https://github.com/pypa/pip/issues/6164 +# even with installing setuptools before upgrading pip ends up with pip seeing +# the older setuptools at the system level if include_system_packages is true +pip>=18.1,<19.0 +# pin Jinja2 and PyYAML to the last versions supporting python 3.4 for trusty +Jinja2<=2.10.1 +PyYAML<=5.2 +setuptools<42 +setuptools-scm<=1.17.0 +charmhelpers>=0.4.0,<1.0.0 +charms.reactive>=0.1.0,<2.0.0 +wheel<0.34 +# pin netaddr to avoid pulling importlib-resources +netaddr<=0.7.19 + +# layer:sshproxy +## +# Copyright 2016 Canonical Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +paramiko>=1.16.0,<1.17 + -- GitLab