Loading nscharm_ns/charms/ns/.build.manifest +38 −23 Original line number Diff line number Diff line Loading @@ -5,15 +5,15 @@ "url": "layer:options" }, { "rev": "1d2489bff56daf2f8d1c06ee9248bb0094e4ad49", "rev": "623e69c7b432456fd4364f6e1835424fd6b5425e", "url": "layer:basic" }, { "rev": "4f3159205af1b3e7a3a532dedcf6b887d2e6d78a", "rev": "d59fe211bb4c2834352dc8bf62b6d6891e81834c", "url": "layer:osm-ns" }, { "rev": "aa34ff8bbe4d847102174d9038068af0f6ebe575", "rev": "b189ed8df5e652bdd886431f3c0220f7f5af87b4", "url": "ns" } ], Loading @@ -28,6 +28,11 @@ "static", "0da5c4dcda27cd6406e5bb81cbf68ddccaf728ac764ec15053a165c1449d87d9" ], ".travis/profile-update.yaml": [ "layer:basic", "static", "731e20aa59bf61c024d317ad630e478301a9386ccc0afe56e6c1c09db07ac83b" ], "LICENSE": [ "layer:basic", "static", Loading Loading @@ -56,7 +61,7 @@ "bin/charm-env": [ "layer:basic", "static", "458c53532c19ee357cbf4209ccc7d811810718ba0ea3b0588b3d3ef040a44b8e" "fb6a20fac4102a6a4b6ffe903fcf666998f9a95a3647e6f9af7a1eeb44e58fd5" ], "bin/layer_option": [ "layer:options", Loading Loading @@ -141,7 +146,7 @@ "layer.yaml": [ "ns", "dynamic", "6e5423e1b33993267456ad163ecf0fcd9e921f99214fbc6f23a1d7902468df8f" "431a730b6fb7851377a021b85ba3ebdfca416448af0bb0618d953928089b1df0" ], "lib/charms/layer/__init__.py": [ "layer:basic", Loading @@ -151,7 +156,7 @@ "lib/charms/layer/basic.py": [ "layer:basic", "static", "445652dbaa1f0b84a7215da185bcbdff097bb9bbbce11b4c2dbbff90f77719a9" "3126b5754ad39402ee27e64527044ddd231ed1cd137fcedaffb51e63a635f108" ], "lib/charms/layer/execd.py": [ "layer:basic", Loading Loading @@ -181,7 +186,7 @@ "reactive/ns.py": [ "ns", "static", "1d1c34c9cdb049552d0c79219cade41e70ae4d76782a305be0e8d4d6928afeb4" "eea5f82e9332f2129f63c8ff528abf3f556c1c68e25fb08fb9cc2255e0ebdcae" ], "reactive/osm_ns.py": [ "layer:osm-ns", Loading @@ -191,7 +196,7 @@ "requirements.txt": [ "layer:basic", "static", "0f1c70d27e26005a96d66ad54482877ae20f7737693c833e29dd72bd6ac24892" "a00f75d80849e5b4fc5ad2e7536f947c25b1a4044b341caa8ee87a92d3a4c804" ], "tests/00-setup": [ "ns", Loading @@ -206,16 +211,21 @@ "version": [ "ns", "dynamic", "260a9a4811db0cebf6acd4bb4910f460278b8c449d7437a7c908ce864fc6a184" "a6dd063a3024e9bf54901a897cd7926d5bceab359e6cb0eb9deb34d2eab29367" ], "wheelhouse/Jinja2-2.10.3.tar.gz": [ "wheelhouse.txt": [ "layer:basic", "dynamic", "9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" "7cf3f983dc8f85b0c0ca6d69accdb4f4af842a911625286df09005ed1897d797" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ "wheelhouse/Jinja2-2.10.1.tar.gz": [ "layer:basic", "dynamic", "065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ "__pip__", "dynamic", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b" ], "wheelhouse/PyYAML-5.2.tar.gz": [ Loading @@ -224,34 +234,39 @@ "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" ], "wheelhouse/Tempita-0.5.2.tar.gz": [ "layer:basic", "__pip__", "dynamic", "cacecf0baa674d356641f1d406b8bff1d756d739c46b869a54de515d08e6fc9c" ], "wheelhouse/charmhelpers-0.20.7.tar.gz": [ "wheelhouse/charmhelpers-0.20.19.tar.gz": [ "layer:basic", "dynamic", "e0f8d005d39cded1b0c5997d8ef1d90832341c67ebeb4a334ad1eb348fbd803a" "74b2d95ec305e5799f0f7d068b0c6e010612ec877feadf32fc51e6bed1f68234" ], "wheelhouse/charms.reactive-1.3.0.tar.gz": [ "wheelhouse/charms.reactive-1.3.2.tar.gz": [ "layer:basic", "dynamic", "82d2c614c82d64bf56e913990f22663e5de64b99db15838abfd9a064f1cc2f51" "f4633eb37143bf9cc2c0e760a67314954c31fa68ec36d5399d386839cb5d54d5" ], "wheelhouse/netaddr-0.7.19.tar.gz": [ "layer:basic", "dynamic", "38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd" ], "wheelhouse/pbr-5.5.1.tar.gz": [ "__pip__", "dynamic", "5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9" ], "wheelhouse/pip-18.1.tar.gz": [ "layer:basic", "dynamic", "c0a292bd977ef590379a3f05d7b7f65135487b67470f6281289a94e015650ea1" ], "wheelhouse/pyaml-19.12.0.tar.gz": [ "layer:basic", "wheelhouse/pyaml-20.4.0.tar.gz": [ "__pip__", "dynamic", "b3f636b467864319d7ded1558f86bb305b8612a274f5d443a62dc5eceb1b7176" "29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71" ], "wheelhouse/setuptools-41.6.0.zip": [ "layer:basic", Loading @@ -263,10 +278,10 @@ "dynamic", "70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a" ], "wheelhouse/six-1.13.0.tar.gz": [ "layer:basic", "wheelhouse/six-1.15.0.tar.gz": [ "__pip__", "dynamic", "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" "30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259" ], "wheelhouse/wheel-0.33.6.tar.gz": [ "layer:basic", Loading nscharm_ns/charms/ns/.travis/profile-update.yaml 0 → 100644 +12 −0 Original line number Diff line number Diff line config: {} description: Default LXD profile - updated devices: eth0: name: eth0 parent: lxdbr0 nictype: bridged type: nic root: path: / pool: default type: disk nscharm_ns/charms/ns/bin/charm-env +3 −3 Original line number Diff line number Diff line Loading @@ -25,7 +25,7 @@ find_charm_dirs() { found_charm_dir="" if [[ -n "$desired_charm" ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do charm_name="$(JUJU_CHARM_DIR="$charm_dir" charm-env python3 -c 'from charmhelpers.core.hookenv import charm_name; print(charm_name())')" charm_name="$(grep -o '^['\''"]\?name['\''"]\?:.*' $charm_dir/metadata.yaml 2> /dev/null | sed -e 's/.*: *//' -e 's/['\''"]//g')" if [[ "$charm_name" == "$desired_charm" ]]; then if [[ -n "$found_charm_dir" ]]; then >&2 echo "Ambiguous possibilities for JUJU_CHARM_DIR matching '$desired_charm'; please run within a Juju hook context" Loading @@ -43,13 +43,13 @@ find_charm_dirs() { return fi # shellcheck disable=SC2126 non_subordinates="$(grep -L 'subordinate:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" non_subordinates="$(grep -L 'subordinate"\?:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" if [[ "$non_subordinates" -gt 1 ]]; then >&2 echo 'Ambiguous possibilities for JUJU_CHARM_DIR; please use --charm or run within a Juju hook context' exit 1 elif [[ "$non_subordinates" -eq 1 ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do if grep -q 'subordinate:.*true' "$charm_dir/metadata.yaml"; then if grep -q 'subordinate"\?:.*true' "$charm_dir/metadata.yaml"; then continue fi export JUJU_CHARM_DIR="$charm_dir" Loading nscharm_ns/charms/ns/layer.yaml +1 −1 Original line number Diff line number Diff line Loading @@ -2,7 +2,7 @@ - "layer:options" - "layer:basic" - "layer:osm-ns" "exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt"] "exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt", "unit_tests"] "options": "basic": "use_venv": !!bool "false" Loading nscharm_ns/charms/ns/lib/charms/layer/basic.py +209 −49 Original line number Diff line number Diff line import os import sys import re import shutil from distutils.version import LooseVersion from pkg_resources import Requirement from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep Loading @@ -9,14 +12,55 @@ from charms import layer from charms.layer.execd import execd_preinstall def lsb_release(): """Return /etc/lsb-release in a dict""" def _get_subprocess_env(): env = os.environ.copy() env['LANG'] = env.get('LANG', 'C.UTF-8') return env def get_series(): """ Return series for a few known OS:es. Tested as of 2019 november: * centos6, centos7, rhel6. * bionic """ series = "" # Looking for content in /etc/os-release # works for ubuntu + some centos if os.path.isfile('/etc/os-release'): d = {} with open('/etc/os-release', 'r') as rel: for l in rel: if not re.match(r'^\s*$', l): k, v = l.split('=') d[k.strip()] = v.strip().replace('"', '') series = "{ID}{VERSION_ID}".format(**d) # Looking for content in /etc/redhat-release # works for redhat enterprise systems elif os.path.isfile('/etc/redhat-release'): with open('/etc/redhat-release', 'r') as redhatlsb: # CentOS Linux release 7.7.1908 (Core) line = redhatlsb.readline() release = int(line.split("release")[1].split()[0][0]) series = "centos" + str(release) # Looking for content in /etc/lsb-release # works for ubuntu elif os.path.isfile('/etc/lsb-release'): d = {} with open('/etc/lsb-release', 'r') as lsb: for l in lsb: k, v = l.split('=') d[k.strip()] = v.strip() return d series = d['DISTRIB_CODENAME'] # This is what happens if we cant figure out the OS. else: series = "unknown" return series def bootstrap_charm_deps(): Loading @@ -30,6 +74,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts series = get_series() # OMG?! is build-essentials needed? ubuntu_packages = ['python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential'] # I'm not going to "yum group info "Development Tools" # omitting above madness centos_packages = ['python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel'] packages_needed = [] if 'centos' in series: packages_needed = centos_packages else: packages_needed = ubuntu_packages charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') Loading @@ -40,8 +108,9 @@ def bootstrap_charm_deps(): is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' post_upgrade = os.path.exists('wheelhouse/.upgrade') is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) is_post_upgrade = os.path.exists('wheelhouse/.upgraded') is_upgrade = (not is_post_upgrade and (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary Loading @@ -50,51 +119,69 @@ def bootstrap_charm_deps(): # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it if post_upgrade: os.unlink('wheelhouse/.upgrade') if is_post_upgrade: os.unlink('wheelhouse/.upgraded') return if is_series_upgrade and os.path.exists(venv): # series upgrade should do a full clear of the venv, rather than just # updating it, to bring in updates to Python itself if os.path.exists(venv): try: # focal installs or upgrades prior to PR 160 could leave the venv # in a broken state which would prevent subsequent charm upgrades _load_installed_versions(vpip) except CalledProcessError: is_broken_venv = True else: is_broken_venv = False if is_upgrade or is_broken_venv: # All upgrades should do a full clear of the venv, rather than # just updating it, to bring in updates to Python itself shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): pre_eoan = series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty', 'ubuntu16.04', 'xenial', 'ubuntu18.04', 'bionic') pydistutils_lines = [ "[easy_install]\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), "no_index=True\n", "index_url=\n", # deliberately nothing here; disables it. ] if pre_eoan: pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) fp.writelines([ "[easy_install]\n", "allow_hosts = ''\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), ]) apt_install([ 'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential', ]) fp.writelines(pydistutils_lines) if 'centos' in series: yum_install(packages_needed) else: apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml if 'centos' in series: yum_install(cfg.get('packages', [])) else: apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): series = lsb_release()['DISTRIB_CODENAME'] if series in ('precise', 'trusty'): series = get_series() if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') check_call(cmd) check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: Loading @@ -103,23 +190,36 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') # need newer pip, to fix spurious Double Requirement error: # https://github.com/pypa/pip/issues/56 check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'pip']) # per https://github.com/juju-solutions/layer-basic/issues/110 # this replaces the setuptools that was copied over from the system on # venv create with latest setuptools and adds setuptools_scm check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'setuptools', 'setuptools-scm']) # install the rest of the wheelhouse deps check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', '-f', 'wheelhouse'] + glob('wheelhouse/*')) pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] # we bundle these packages to work around bugs in older versions (such # as https://github.com/pypa/pip/issues/56), but if the system already # provided a newer version, downgrading it can cause other problems _update_if_newer(pip, pre_install_pkgs) # install the rest of the wheelhouse deps (extract the pkg names into # a set so that we can ignore the pre-install packages and let pip # choose the best version in case there are multiple from layer # conflicts) pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) reinstall_flag = '--force-reinstall' if not cfg.get('use_venv', True) and pre_eoan: reinstall_flag = '--ignore-installed' check_call([pip, 'install', '-U', reinstall_flag, '--no-index', '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') # install pyyaml for centos7, since, unlike the ubuntu image, the # default image for centos doesn't include pyyaml; see the discussion: # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin if 'centos' in series: check_call([pip, 'install', '-U', 'pyyaml'], env=_get_subprocess_env()) # install python packages from layer options if cfg.get('python_packages'): check_call([pip, 'install', '-U'] + cfg.get('python_packages')) check_call([pip, 'install', '-U'] + cfg.get('python_packages'), env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it Loading @@ -144,6 +244,9 @@ def bootstrap_charm_deps(): os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() if is_upgrade: # flag us as having already upgraded so we don't do it again open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available Loading @@ -151,6 +254,39 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) def _load_installed_versions(pip): pip_freeze = check_output([pip, 'freeze']).decode('utf8') versions = {} for pkg_ver in pip_freeze.splitlines(): try: req = Requirement.parse(pkg_ver) except ValueError: continue versions.update({ req.project_name: LooseVersion(ver) for op, ver in req.specs if op == '==' }) return versions def _load_wheelhouse_versions(): versions = {} for wheel in glob('wheelhouse/*'): pkg, ver = os.path.basename(wheel).rsplit('-', 1) # nb: LooseVersion ignores the file extension versions[pkg.replace('_', '-')] = LooseVersion(ver) return versions def _update_if_newer(pip, pkgs): installed = _load_installed_versions(pip) wheelhouse = _load_wheelhouse_versions() for pkg in pkgs: if pkg not in installed or wheelhouse[pkg] > installed[pkg]: check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', pkg], env=_get_subprocess_env()) def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: Loading Loading @@ -225,7 +361,7 @@ def apt_install(packages): if isinstance(packages, (str, bytes)): packages = [packages] env = os.environ.copy() env = _get_subprocess_env() if 'DEBIAN_FRONTEND' not in env: env['DEBIAN_FRONTEND'] = 'noninteractive' Loading @@ -242,7 +378,7 @@ def apt_install(packages): raise try: # sometimes apt-get update needs to be run check_call(['apt-get', 'update']) check_call(['apt-get', 'update'], env=env) except CalledProcessError: # sometimes it's a dpkg lock issue pass Loading @@ -251,6 +387,30 @@ def apt_install(packages): break def yum_install(packages): """ Installs packages with yum. This function largely mimics the apt_install function for consistency. """ if packages: env = os.environ.copy() cmd = ['yum', '-y', 'install'] for attempt in range(3): try: check_call(cmd + packages, env=env) except CalledProcessError: if attempt == 2: raise try: check_call(['yum', 'update'], env=env) except CalledProcessError: pass sleep(5) else: break else: pass def init_config_states(): import yaml from charmhelpers.core import hookenv Loading Loading
nscharm_ns/charms/ns/.build.manifest +38 −23 Original line number Diff line number Diff line Loading @@ -5,15 +5,15 @@ "url": "layer:options" }, { "rev": "1d2489bff56daf2f8d1c06ee9248bb0094e4ad49", "rev": "623e69c7b432456fd4364f6e1835424fd6b5425e", "url": "layer:basic" }, { "rev": "4f3159205af1b3e7a3a532dedcf6b887d2e6d78a", "rev": "d59fe211bb4c2834352dc8bf62b6d6891e81834c", "url": "layer:osm-ns" }, { "rev": "aa34ff8bbe4d847102174d9038068af0f6ebe575", "rev": "b189ed8df5e652bdd886431f3c0220f7f5af87b4", "url": "ns" } ], Loading @@ -28,6 +28,11 @@ "static", "0da5c4dcda27cd6406e5bb81cbf68ddccaf728ac764ec15053a165c1449d87d9" ], ".travis/profile-update.yaml": [ "layer:basic", "static", "731e20aa59bf61c024d317ad630e478301a9386ccc0afe56e6c1c09db07ac83b" ], "LICENSE": [ "layer:basic", "static", Loading Loading @@ -56,7 +61,7 @@ "bin/charm-env": [ "layer:basic", "static", "458c53532c19ee357cbf4209ccc7d811810718ba0ea3b0588b3d3ef040a44b8e" "fb6a20fac4102a6a4b6ffe903fcf666998f9a95a3647e6f9af7a1eeb44e58fd5" ], "bin/layer_option": [ "layer:options", Loading Loading @@ -141,7 +146,7 @@ "layer.yaml": [ "ns", "dynamic", "6e5423e1b33993267456ad163ecf0fcd9e921f99214fbc6f23a1d7902468df8f" "431a730b6fb7851377a021b85ba3ebdfca416448af0bb0618d953928089b1df0" ], "lib/charms/layer/__init__.py": [ "layer:basic", Loading @@ -151,7 +156,7 @@ "lib/charms/layer/basic.py": [ "layer:basic", "static", "445652dbaa1f0b84a7215da185bcbdff097bb9bbbce11b4c2dbbff90f77719a9" "3126b5754ad39402ee27e64527044ddd231ed1cd137fcedaffb51e63a635f108" ], "lib/charms/layer/execd.py": [ "layer:basic", Loading Loading @@ -181,7 +186,7 @@ "reactive/ns.py": [ "ns", "static", "1d1c34c9cdb049552d0c79219cade41e70ae4d76782a305be0e8d4d6928afeb4" "eea5f82e9332f2129f63c8ff528abf3f556c1c68e25fb08fb9cc2255e0ebdcae" ], "reactive/osm_ns.py": [ "layer:osm-ns", Loading @@ -191,7 +196,7 @@ "requirements.txt": [ "layer:basic", "static", "0f1c70d27e26005a96d66ad54482877ae20f7737693c833e29dd72bd6ac24892" "a00f75d80849e5b4fc5ad2e7536f947c25b1a4044b341caa8ee87a92d3a4c804" ], "tests/00-setup": [ "ns", Loading @@ -206,16 +211,21 @@ "version": [ "ns", "dynamic", "260a9a4811db0cebf6acd4bb4910f460278b8c449d7437a7c908ce864fc6a184" "a6dd063a3024e9bf54901a897cd7926d5bceab359e6cb0eb9deb34d2eab29367" ], "wheelhouse/Jinja2-2.10.3.tar.gz": [ "wheelhouse.txt": [ "layer:basic", "dynamic", "9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" "7cf3f983dc8f85b0c0ca6d69accdb4f4af842a911625286df09005ed1897d797" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ "wheelhouse/Jinja2-2.10.1.tar.gz": [ "layer:basic", "dynamic", "065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013" ], "wheelhouse/MarkupSafe-1.1.1.tar.gz": [ "__pip__", "dynamic", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b" ], "wheelhouse/PyYAML-5.2.tar.gz": [ Loading @@ -224,34 +234,39 @@ "c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c" ], "wheelhouse/Tempita-0.5.2.tar.gz": [ "layer:basic", "__pip__", "dynamic", "cacecf0baa674d356641f1d406b8bff1d756d739c46b869a54de515d08e6fc9c" ], "wheelhouse/charmhelpers-0.20.7.tar.gz": [ "wheelhouse/charmhelpers-0.20.19.tar.gz": [ "layer:basic", "dynamic", "e0f8d005d39cded1b0c5997d8ef1d90832341c67ebeb4a334ad1eb348fbd803a" "74b2d95ec305e5799f0f7d068b0c6e010612ec877feadf32fc51e6bed1f68234" ], "wheelhouse/charms.reactive-1.3.0.tar.gz": [ "wheelhouse/charms.reactive-1.3.2.tar.gz": [ "layer:basic", "dynamic", "82d2c614c82d64bf56e913990f22663e5de64b99db15838abfd9a064f1cc2f51" "f4633eb37143bf9cc2c0e760a67314954c31fa68ec36d5399d386839cb5d54d5" ], "wheelhouse/netaddr-0.7.19.tar.gz": [ "layer:basic", "dynamic", "38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd" ], "wheelhouse/pbr-5.5.1.tar.gz": [ "__pip__", "dynamic", "5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9" ], "wheelhouse/pip-18.1.tar.gz": [ "layer:basic", "dynamic", "c0a292bd977ef590379a3f05d7b7f65135487b67470f6281289a94e015650ea1" ], "wheelhouse/pyaml-19.12.0.tar.gz": [ "layer:basic", "wheelhouse/pyaml-20.4.0.tar.gz": [ "__pip__", "dynamic", "b3f636b467864319d7ded1558f86bb305b8612a274f5d443a62dc5eceb1b7176" "29a5c2a68660a799103d6949167bd6c7953d031449d08802386372de1db6ad71" ], "wheelhouse/setuptools-41.6.0.zip": [ "layer:basic", Loading @@ -263,10 +278,10 @@ "dynamic", "70a4cf5584e966ae92f54a764e6437af992ba42ac4bca7eb37cc5d02b98ec40a" ], "wheelhouse/six-1.13.0.tar.gz": [ "layer:basic", "wheelhouse/six-1.15.0.tar.gz": [ "__pip__", "dynamic", "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" "30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259" ], "wheelhouse/wheel-0.33.6.tar.gz": [ "layer:basic", Loading
nscharm_ns/charms/ns/.travis/profile-update.yaml 0 → 100644 +12 −0 Original line number Diff line number Diff line config: {} description: Default LXD profile - updated devices: eth0: name: eth0 parent: lxdbr0 nictype: bridged type: nic root: path: / pool: default type: disk
nscharm_ns/charms/ns/bin/charm-env +3 −3 Original line number Diff line number Diff line Loading @@ -25,7 +25,7 @@ find_charm_dirs() { found_charm_dir="" if [[ -n "$desired_charm" ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do charm_name="$(JUJU_CHARM_DIR="$charm_dir" charm-env python3 -c 'from charmhelpers.core.hookenv import charm_name; print(charm_name())')" charm_name="$(grep -o '^['\''"]\?name['\''"]\?:.*' $charm_dir/metadata.yaml 2> /dev/null | sed -e 's/.*: *//' -e 's/['\''"]//g')" if [[ "$charm_name" == "$desired_charm" ]]; then if [[ -n "$found_charm_dir" ]]; then >&2 echo "Ambiguous possibilities for JUJU_CHARM_DIR matching '$desired_charm'; please run within a Juju hook context" Loading @@ -43,13 +43,13 @@ find_charm_dirs() { return fi # shellcheck disable=SC2126 non_subordinates="$(grep -L 'subordinate:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" non_subordinates="$(grep -L 'subordinate"\?:.*true' "$agents_dir"/unit-*/charm/metadata.yaml | wc -l)" if [[ "$non_subordinates" -gt 1 ]]; then >&2 echo 'Ambiguous possibilities for JUJU_CHARM_DIR; please use --charm or run within a Juju hook context' exit 1 elif [[ "$non_subordinates" -eq 1 ]]; then for charm_dir in $(/bin/ls -d "$agents_dir"/unit-*/charm); do if grep -q 'subordinate:.*true' "$charm_dir/metadata.yaml"; then if grep -q 'subordinate"\?:.*true' "$charm_dir/metadata.yaml"; then continue fi export JUJU_CHARM_DIR="$charm_dir" Loading
nscharm_ns/charms/ns/layer.yaml +1 −1 Original line number Diff line number Diff line Loading @@ -2,7 +2,7 @@ - "layer:options" - "layer:basic" - "layer:osm-ns" "exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt"] "exclude": [".travis.yml", "tests", "tox.ini", "test-requirements.txt", "unit_tests"] "options": "basic": "use_venv": !!bool "false" Loading
nscharm_ns/charms/ns/lib/charms/layer/basic.py +209 −49 Original line number Diff line number Diff line import os import sys import re import shutil from distutils.version import LooseVersion from pkg_resources import Requirement from glob import glob from subprocess import check_call, check_output, CalledProcessError from time import sleep Loading @@ -9,14 +12,55 @@ from charms import layer from charms.layer.execd import execd_preinstall def lsb_release(): """Return /etc/lsb-release in a dict""" def _get_subprocess_env(): env = os.environ.copy() env['LANG'] = env.get('LANG', 'C.UTF-8') return env def get_series(): """ Return series for a few known OS:es. Tested as of 2019 november: * centos6, centos7, rhel6. * bionic """ series = "" # Looking for content in /etc/os-release # works for ubuntu + some centos if os.path.isfile('/etc/os-release'): d = {} with open('/etc/os-release', 'r') as rel: for l in rel: if not re.match(r'^\s*$', l): k, v = l.split('=') d[k.strip()] = v.strip().replace('"', '') series = "{ID}{VERSION_ID}".format(**d) # Looking for content in /etc/redhat-release # works for redhat enterprise systems elif os.path.isfile('/etc/redhat-release'): with open('/etc/redhat-release', 'r') as redhatlsb: # CentOS Linux release 7.7.1908 (Core) line = redhatlsb.readline() release = int(line.split("release")[1].split()[0][0]) series = "centos" + str(release) # Looking for content in /etc/lsb-release # works for ubuntu elif os.path.isfile('/etc/lsb-release'): d = {} with open('/etc/lsb-release', 'r') as lsb: for l in lsb: k, v = l.split('=') d[k.strip()] = v.strip() return d series = d['DISTRIB_CODENAME'] # This is what happens if we cant figure out the OS. else: series = "unknown" return series def bootstrap_charm_deps(): Loading @@ -30,6 +74,30 @@ def bootstrap_charm_deps(): # unless the operator has created and populated $JUJU_CHARM_DIR/exec.d. execd_preinstall() # ensure that $JUJU_CHARM_DIR/bin is on the path, for helper scripts series = get_series() # OMG?! is build-essentials needed? ubuntu_packages = ['python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential'] # I'm not going to "yum group info "Development Tools" # omitting above madness centos_packages = ['python3-pip', 'python3-setuptools', 'python3-devel', 'python3-wheel'] packages_needed = [] if 'centos' in series: packages_needed = centos_packages else: packages_needed = ubuntu_packages charm_dir = os.environ['JUJU_CHARM_DIR'] os.environ['PATH'] += ':%s' % os.path.join(charm_dir, 'bin') venv = os.path.abspath('../.venv') Loading @@ -40,8 +108,9 @@ def bootstrap_charm_deps(): is_bootstrapped = os.path.exists('wheelhouse/.bootstrapped') is_charm_upgrade = hook_name == 'upgrade-charm' is_series_upgrade = hook_name == 'post-series-upgrade' post_upgrade = os.path.exists('wheelhouse/.upgrade') is_upgrade = not post_upgrade and (is_charm_upgrade or is_series_upgrade) is_post_upgrade = os.path.exists('wheelhouse/.upgraded') is_upgrade = (not is_post_upgrade and (is_charm_upgrade or is_series_upgrade)) if is_bootstrapped and not is_upgrade: # older subordinates might have downgraded charm-env, so we should # restore it if necessary Loading @@ -50,51 +119,69 @@ def bootstrap_charm_deps(): # the .upgrade file prevents us from getting stuck in a loop # when re-execing to activate the venv; at this point, we've # activated the venv, so it's safe to clear it if post_upgrade: os.unlink('wheelhouse/.upgrade') if is_post_upgrade: os.unlink('wheelhouse/.upgraded') return if is_series_upgrade and os.path.exists(venv): # series upgrade should do a full clear of the venv, rather than just # updating it, to bring in updates to Python itself if os.path.exists(venv): try: # focal installs or upgrades prior to PR 160 could leave the venv # in a broken state which would prevent subsequent charm upgrades _load_installed_versions(vpip) except CalledProcessError: is_broken_venv = True else: is_broken_venv = False if is_upgrade or is_broken_venv: # All upgrades should do a full clear of the venv, rather than # just updating it, to bring in updates to Python itself shutil.rmtree(venv) if is_upgrade: if os.path.exists('wheelhouse/.bootstrapped'): os.unlink('wheelhouse/.bootstrapped') open('wheelhouse/.upgrade', 'w').close() # bootstrap wheelhouse if os.path.exists('wheelhouse'): pre_eoan = series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty', 'ubuntu16.04', 'xenial', 'ubuntu18.04', 'bionic') pydistutils_lines = [ "[easy_install]\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), "no_index=True\n", "index_url=\n", # deliberately nothing here; disables it. ] if pre_eoan: pydistutils_lines.append("allow_hosts = ''\n") with open('/root/.pydistutils.cfg', 'w') as fp: # make sure that easy_install also only uses the wheelhouse # (see https://github.com/pypa/pip/issues/410) fp.writelines([ "[easy_install]\n", "allow_hosts = ''\n", "find_links = file://{}/wheelhouse/\n".format(charm_dir), ]) apt_install([ 'python3-pip', 'python3-setuptools', 'python3-yaml', 'python3-dev', 'python3-wheel', 'build-essential', ]) fp.writelines(pydistutils_lines) if 'centos' in series: yum_install(packages_needed) else: apt_install(packages_needed) from charms.layer import options cfg = options.get('basic') # include packages defined in layer.yaml if 'centos' in series: yum_install(cfg.get('packages', [])) else: apt_install(cfg.get('packages', [])) # if we're using a venv, set it up if cfg.get('use_venv'): if not os.path.exists(venv): series = lsb_release()['DISTRIB_CODENAME'] if series in ('precise', 'trusty'): series = get_series() if series in ('ubuntu12.04', 'precise', 'ubuntu14.04', 'trusty'): apt_install(['python-virtualenv']) elif 'centos' in series: yum_install(['python-virtualenv']) else: apt_install(['virtualenv']) cmd = ['virtualenv', '-ppython3', '--never-download', venv] if cfg.get('include_system_packages'): cmd.append('--system-site-packages') check_call(cmd) check_call(cmd, env=_get_subprocess_env()) os.environ['PATH'] = ':'.join([vbin, os.environ['PATH']]) pip = vpip else: Loading @@ -103,23 +190,36 @@ def bootstrap_charm_deps(): # from changing it if os.path.exists('/usr/bin/pip'): shutil.copy2('/usr/bin/pip', '/usr/bin/pip.save') # need newer pip, to fix spurious Double Requirement error: # https://github.com/pypa/pip/issues/56 check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'pip']) # per https://github.com/juju-solutions/layer-basic/issues/110 # this replaces the setuptools that was copied over from the system on # venv create with latest setuptools and adds setuptools_scm check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', 'setuptools', 'setuptools-scm']) # install the rest of the wheelhouse deps check_call([pip, 'install', '-U', '--ignore-installed', '--no-index', '-f', 'wheelhouse'] + glob('wheelhouse/*')) pre_install_pkgs = ['pip', 'setuptools', 'setuptools-scm'] # we bundle these packages to work around bugs in older versions (such # as https://github.com/pypa/pip/issues/56), but if the system already # provided a newer version, downgrading it can cause other problems _update_if_newer(pip, pre_install_pkgs) # install the rest of the wheelhouse deps (extract the pkg names into # a set so that we can ignore the pre-install packages and let pip # choose the best version in case there are multiple from layer # conflicts) pkgs = _load_wheelhouse_versions().keys() - set(pre_install_pkgs) reinstall_flag = '--force-reinstall' if not cfg.get('use_venv', True) and pre_eoan: reinstall_flag = '--ignore-installed' check_call([pip, 'install', '-U', reinstall_flag, '--no-index', '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), env=_get_subprocess_env()) # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') # install pyyaml for centos7, since, unlike the ubuntu image, the # default image for centos doesn't include pyyaml; see the discussion: # https://discourse.jujucharms.com/t/charms-for-centos-lets-begin if 'centos' in series: check_call([pip, 'install', '-U', 'pyyaml'], env=_get_subprocess_env()) # install python packages from layer options if cfg.get('python_packages'): check_call([pip, 'install', '-U'] + cfg.get('python_packages')) check_call([pip, 'install', '-U'] + cfg.get('python_packages'), env=_get_subprocess_env()) if not cfg.get('use_venv'): # restore system pip to prevent `pip3 install -U pip` # from changing it Loading @@ -144,6 +244,9 @@ def bootstrap_charm_deps(): os.symlink('/usr/local/sbin/layer_option', 'bin/layer_option') # flag us as having already bootstrapped so we don't do it again open('wheelhouse/.bootstrapped', 'w').close() if is_upgrade: # flag us as having already upgraded so we don't do it again open('wheelhouse/.upgraded', 'w').close() # Ensure that the newly bootstrapped libs are available. # Note: this only seems to be an issue with namespace packages. # Non-namespace-package libs (e.g., charmhelpers) are available Loading @@ -151,6 +254,39 @@ def bootstrap_charm_deps(): reload_interpreter(vpy if cfg.get('use_venv') else sys.argv[0]) def _load_installed_versions(pip): pip_freeze = check_output([pip, 'freeze']).decode('utf8') versions = {} for pkg_ver in pip_freeze.splitlines(): try: req = Requirement.parse(pkg_ver) except ValueError: continue versions.update({ req.project_name: LooseVersion(ver) for op, ver in req.specs if op == '==' }) return versions def _load_wheelhouse_versions(): versions = {} for wheel in glob('wheelhouse/*'): pkg, ver = os.path.basename(wheel).rsplit('-', 1) # nb: LooseVersion ignores the file extension versions[pkg.replace('_', '-')] = LooseVersion(ver) return versions def _update_if_newer(pip, pkgs): installed = _load_installed_versions(pip) wheelhouse = _load_wheelhouse_versions() for pkg in pkgs: if pkg not in installed or wheelhouse[pkg] > installed[pkg]: check_call([pip, 'install', '-U', '--no-index', '-f', 'wheelhouse', pkg], env=_get_subprocess_env()) def install_or_update_charm_env(): # On Trusty python3-pkg-resources is not installed try: Loading Loading @@ -225,7 +361,7 @@ def apt_install(packages): if isinstance(packages, (str, bytes)): packages = [packages] env = os.environ.copy() env = _get_subprocess_env() if 'DEBIAN_FRONTEND' not in env: env['DEBIAN_FRONTEND'] = 'noninteractive' Loading @@ -242,7 +378,7 @@ def apt_install(packages): raise try: # sometimes apt-get update needs to be run check_call(['apt-get', 'update']) check_call(['apt-get', 'update'], env=env) except CalledProcessError: # sometimes it's a dpkg lock issue pass Loading @@ -251,6 +387,30 @@ def apt_install(packages): break def yum_install(packages): """ Installs packages with yum. This function largely mimics the apt_install function for consistency. """ if packages: env = os.environ.copy() cmd = ['yum', '-y', 'install'] for attempt in range(3): try: check_call(cmd + packages, env=env) except CalledProcessError: if attempt == 2: raise try: check_call(['yum', 'update'], env=env) except CalledProcessError: pass sleep(5) else: break else: pass def init_config_states(): import yaml from charmhelpers.core import hookenv Loading
