From: sousaedu Date: Fri, 15 Jan 2021 16:59:14 +0000 (+0000) Subject: Adding Prometheus charm X-Git-Tag: branch-sol006v331-start~82 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fdevops.git;a=commitdiff_plain;h=89f86a35ba689fb6706020008372075b16aa21a6 Adding Prometheus charm Change-Id: Ib17db114e32c2cbaaa89474e92890263f705e0e5 Signed-off-by: sousaedu --- diff --git a/installers/charm/prometheus-k8s/.gitignore b/installers/charm/prometheus-k8s/.gitignore new file mode 100644 index 00000000..712eb963 --- /dev/null +++ b/installers/charm/prometheus-k8s/.gitignore @@ -0,0 +1,24 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +release/ +__pycache__ +.tox diff --git a/installers/charm/prometheus-k8s/.yamllint.yaml b/installers/charm/prometheus-k8s/.yamllint.yaml new file mode 100644 index 00000000..21b95b5b --- /dev/null +++ b/installers/charm/prometheus-k8s/.yamllint.yaml @@ -0,0 +1,34 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +--- + +extends: default +rules: + line-length: disable +yaml-files: + - '*.yaml' + - '*.yml' + - '.yamllint' +ignore: | + reactive/ + .tox + release/ diff --git a/installers/charm/prometheus-k8s/README.md b/installers/charm/prometheus-k8s/README.md new file mode 100755 index 00000000..e6ae8017 --- /dev/null +++ b/installers/charm/prometheus-k8s/README.md @@ -0,0 +1,108 @@ + + +# Overview + +Prometheus for Juju CAAS + +## Testing + +The tests of this charm are done using tox and Zaza. + + + +### Prepare environment + +The machine in which the tests are run needs access to a juju k8s controller. The easiest way to approach this is by executing the following commands: + +``` +sudo apt install tox -y +sudo snap install microk8s --classic +sudo snap install juju + +microk8s.status --wait-ready +microk8s.enable storage dashboard dns + +juju bootstrap microk8s k8s-cloud +``` + +If /usr/bin/python does not exist, you should probably need to do this: + +``` +sudo ln -s /usr/bin/python3 /usr/bin/python +``` +### Build Charm + +**Download dependencies:** + +``` +mkdir -p ~/charm/layers ~/charm/build ~/charm/interfaces +git clone https://git.launchpad.net/canonical-osm ~/canonical-osm + +cd ~/charm/layers +git clone https://git.launchpad.net/charm-k8s-prometheus prometheus-k8s + +cd ~/charm/interfaces +mv ~/canonical-osm/charms/interfaces/* . +``` + +**Charm structure:** + +``` +├── config.yaml +├── icon.svg +├── layer.yaml +├── metadata.yaml +├── reactive +│ ├── spec_template.yaml +│ └── prometheus.py +├── README.md +├── test-requirements.txt +├── tests +│ ├── basic_deployment.py +│ ├── bundles +│ │ ├── prometheus-ha.yaml +│ │ └── prometheus.yaml +│ └── tests.yaml +└── tox.ini +``` + +**Setup environment variables:** + +``` +export CHARM_LAYERS_DIR=~/charm/layers +export CHARM_BUILD_DIR=~/charm/build +export CHARM_INTERFACES_DIR=~/charm/interfaces +``` + +**Build:** + +``` +mkdir ~/charm/layers/prometheus-k8s/tests/build +charm build ~/charm/layers/prometheus-k8s +mv ~/charm/build/* ~/charm/layers/prometheus-k8s/tests/build/ +``` + +### Test charm with Tox + +``` +cd ~/charm/layers/prometheus-k8s +tox -e func +``` diff --git a/installers/charm/prometheus-k8s/config.yaml b/installers/charm/prometheus-k8s/config.yaml new file mode 100755 index 00000000..3c3e9c6c --- /dev/null +++ b/installers/charm/prometheus-k8s/config.yaml @@ -0,0 +1,46 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +options: + advertised-hostname: + description: Advertised hostname + type: string + default: "prometheus" + advertised-port: + description: Prometheus Port + type: int + default: 9090 + web-subpath: + description: Subpath for accessing Prometheus + type: string + default: / + default-target: + description: Default target to be added in Prometheus + type: string + default: "" + prometheus-image: + type: string + description: OCI image + default: rocks.canonical.com:443/prom/prometheus:latest + alpine-image: + type: string + description: OCI image + default: rocks.canonical.com:443/alpine:latest diff --git a/installers/charm/prometheus-k8s/icon.svg b/installers/charm/prometheus-k8s/icon.svg new file mode 100644 index 00000000..ffa6296e --- /dev/null +++ b/installers/charm/prometheus-k8s/icon.svg @@ -0,0 +1,12 @@ + + + + prometheus + Created with Sketch. + + + + + + + \ No newline at end of file diff --git a/installers/charm/prometheus-k8s/layer.yaml b/installers/charm/prometheus-k8s/layer.yaml new file mode 100644 index 00000000..b58fad55 --- /dev/null +++ b/installers/charm/prometheus-k8s/layer.yaml @@ -0,0 +1,28 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +includes: + - "layer:caas-base" + - 'layer:status' + - 'layer:leadership' + - 'interface:prometheus' + +repo: https://code.launchpad.net/osm-k8s-bundle diff --git a/installers/charm/prometheus-k8s/metadata.yaml b/installers/charm/prometheus-k8s/metadata.yaml new file mode 100755 index 00000000..78aa7788 --- /dev/null +++ b/installers/charm/prometheus-k8s/metadata.yaml @@ -0,0 +1,41 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +name: "prometheus-k8s" +summary: "Prometheus Charm" +maintainers: + - "SolutionsQA " +description: | + A CAAS charm to deploy Prometheus. +tags: + - "application" +series: + - "kubernetes" +provides: + prometheus: + interface: prometheus +storage: + database: + type: filesystem + location: /prometheus +deployment: + type: stateful + service: cluster diff --git a/installers/charm/prometheus-k8s/reactive/prometheus.py b/installers/charm/prometheus-k8s/reactive/prometheus.py new file mode 100644 index 00000000..9aa4f26e --- /dev/null +++ b/installers/charm/prometheus-k8s/reactive/prometheus.py @@ -0,0 +1,107 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +from charms.reactive import when, when_not, hook +from charms.reactive.flags import set_flag, clear_flag +from charmhelpers.core.hookenv import ( + log, + metadata, + config, + network_get, + relation_id, +) +from charms import layer +from charmhelpers.core import hookenv +import traceback + + +@hook("upgrade-charm") +@when("leadership.is_leader") +def upgrade(): + clear_flag("prometheus-k8s.configured") + + +@when("config.changed") +@when("leadership.is_leader") +def restart(): + clear_flag("prometheus-k8s.configured") + + +@when_not("prometheus-k8s.configured") +@when("leadership.is_leader") +def configure(): + layer.status.maintenance("Configuring prometheus container") + try: + spec = make_pod_spec() + log("set pod spec:\n{}".format(spec)) + layer.caas_base.pod_spec_set(spec) + set_flag("prometheus-k8s.configured") + layer.status.active("ready") + + except Exception as e: + layer.status.blocked("k8s spec failed to deploy: {}".format(e)) + log(traceback.format_exc(), level=hookenv.ERROR) + + +@when("prometheus-k8s.configured") +def set_prometheus_active(): + layer.status.active("ready") + + +@when_not("leadership.is_leader") +def non_leaders_active(): + layer.status.active("ready") + + +@when("prometheus-k8s.configured", "endpoint.prometheus.available") +def send_config(prometheus): + layer.status.maintenance("Sending prometheus configuration") + cfg = config() + try: + info = network_get("prometheus", relation_id()) + log("network info {0}".format(info)) + host = info.get("ingress-addresses", [""])[0] + + prometheus.configure(hostname=host, port=cfg.get("advertised-port")) + clear_flag("endpoint.prometheus.available") + except Exception as e: + log("Exception sending config: {}".format(e)) + + +def make_pod_spec(): + """Make pod specification for Kubernetes + + Returns: + pod_spec: Pod specification for Kubernetes + """ + with open("reactive/spec_template.yaml") as spec_file: + pod_spec_template = spec_file.read() + + md = metadata() + cfg = config() + + data = { + "name": md.get("name"), + "docker_image": cfg.get("prometheus-image"), + "a_docker_image": cfg.get("alpine-image"), + } + data.update(cfg) + return pod_spec_template % data diff --git a/installers/charm/prometheus-k8s/reactive/spec_template.yaml b/installers/charm/prometheus-k8s/reactive/spec_template.yaml new file mode 100644 index 00000000..018b74f1 --- /dev/null +++ b/installers/charm/prometheus-k8s/reactive/spec_template.yaml @@ -0,0 +1,68 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +version: 2 +containers: + - name: "init-chown-data" + image: %(a_docker_image)s + imagePullPolicy: "IfNotPresent" + command: ["/bin/sh", "-c", "chown -R 65534:65534 /prometheus && while true; do sleep 86400; done"] + - name: %(name)s + image: %(docker_image)s + ports: + - containerPort: %(advertised-port)s + protocol: TCP + config: + ALLOW_ANONYMOUS_LOGIN: 'yes' + kubernetes: + readinessProbe: + httpGet: + path: /-/ready + port: %(advertised-port)s + initialDelaySeconds: 10 + timeoutSeconds: 30 + livenessProbe: + httpGet: + path: /-/healthy + port: %(advertised-port)s + initialDelaySeconds: 30 + timeoutSeconds: 30 + command: + - "sh" + - "-c" + - "/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles --web.route-prefix=%(web-subpath)s --web.external-url=http://localhost:%(advertised-port)s%(web-subpath)s" + files: + - name: config + mountPath: /etc/prometheus + files: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + alerting: + alertmanagers: + - static_configs: + - targets: + rule_files: + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['%(default-target)s'] diff --git a/installers/charm/prometheus-k8s/test-requirements.txt b/installers/charm/prometheus-k8s/test-requirements.txt new file mode 100644 index 00000000..b302c2e7 --- /dev/null +++ b/installers/charm/prometheus-k8s/test-requirements.txt @@ -0,0 +1,22 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +git+https://github.com/davigar15/zaza.git#egg=zaza diff --git a/installers/charm/prometheus-k8s/tests/basic_deployment.py b/installers/charm/prometheus-k8s/tests/basic_deployment.py new file mode 100644 index 00000000..5adfb244 --- /dev/null +++ b/installers/charm/prometheus-k8s/tests/basic_deployment.py @@ -0,0 +1,61 @@ +#!/usr/bin/python3 +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +import unittest +import zaza.model as model +import requests as http + + +def get_prometheus_uri(): + ip = model.get_status().applications["prometheus-k8s"]["public-address"] + port = 9090 + return "http://{}:{}".format(ip, port) + + +class BasicDeployment(unittest.TestCase): + def test_get_prometheus_uri(self): + get_prometheus_uri() + + def test_prometheus_get_series(self): + prometheus_uri = get_prometheus_uri() + body = http.get("{}/api/v1/series?match[]=up".format(prometheus_uri)) + self.assertEqual(body.status_code, 200) + + def test_prometheus_get_labels(self): + prometheus_uri = get_prometheus_uri() + body = http.get("{}/api/v1/labels".format(prometheus_uri)) + self.assertEqual(body.status_code, 200) + + def test_prometheus_get_targets(self): + prometheus_uri = get_prometheus_uri() + body = http.get("{}/api/v1/targets".format(prometheus_uri)) + self.assertEqual(body.status_code, 200) + + def test_prometheus_get_alerts(self): + prometheus_uri = get_prometheus_uri() + body = http.get("{}/api/v1/alerts".format(prometheus_uri)) + self.assertEqual(body.status_code, 200) + + def test_prometheus_get_status_config(self): + prometheus_uri = get_prometheus_uri() + body = http.get("{}/api/v1/status/config".format(prometheus_uri)) + self.assertEqual(body.status_code, 200) diff --git a/installers/charm/prometheus-k8s/tests/bundles/prometheus-ha.yaml b/installers/charm/prometheus-k8s/tests/bundles/prometheus-ha.yaml new file mode 100644 index 00000000..fa661660 --- /dev/null +++ b/installers/charm/prometheus-k8s/tests/bundles/prometheus-ha.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +bundle: kubernetes +applications: + prometheus-k8s: + charm: '../../release' + scale: 2 + options: + default-target: "mon-k8s:8000" + series: kubernetes + storage: + database: 50M diff --git a/installers/charm/prometheus-k8s/tests/bundles/prometheus.yaml b/installers/charm/prometheus-k8s/tests/bundles/prometheus.yaml new file mode 100644 index 00000000..957cabdd --- /dev/null +++ b/installers/charm/prometheus-k8s/tests/bundles/prometheus.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +bundle: kubernetes +applications: + prometheus-k8s: + charm: '../../release' + scale: 1 + options: + default-target: "mon-k8s:8000" + series: kubernetes + storage: + database: 50M diff --git a/installers/charm/prometheus-k8s/tests/tests.yaml b/installers/charm/prometheus-k8s/tests/tests.yaml new file mode 100644 index 00000000..f5581a35 --- /dev/null +++ b/installers/charm/prometheus-k8s/tests/tests.yaml @@ -0,0 +1,28 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +gate_bundles: + - prometheus + - prometheus-ha +smoke_bundles: + - prometheus +tests: + - tests.basic_deployment.BasicDeployment diff --git a/installers/charm/prometheus-k8s/tox.ini b/installers/charm/prometheus-k8s/tox.ini new file mode 100644 index 00000000..c242072f --- /dev/null +++ b/installers/charm/prometheus-k8s/tox.ini @@ -0,0 +1,82 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: legal@canonical.com +# +# To get in touch with the maintainers, please contact: +# osm-charmers@lists.launchpad.net +## + +[tox] +envlist = pep8 +skipsdist = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 +whitelist_externals = juju +passenv = HOME TERM CS_API_* OS_* AMULET_* +deps = -r{toxinidir}/test-requirements.txt +install_command = + pip install {opts} {packages} + +[testenv:build] +basepython = python3 +passenv=HTTP_PROXY HTTPS_PROXY NO_PROXY +setenv = CHARM_LAYERS_DIR = /tmp +whitelist_externals = git + charm + rm + mv +commands = + rm -rf /tmp/canonical-osm /tmp/osm-common + rm -rf release/ + git clone https://git.launchpad.net/charm-osm-common /tmp/osm-common + charm build . --build-dir /tmp + mv /tmp/prometheus-k8s/ release/ + +[testenv:black] +basepython = python3 +deps = + black + yamllint + flake8 +commands = + black --check --diff . + yamllint . + flake8 reactive/ --max-line-length=88 + flake8 tests/ --max-line-length=88 + +[testenv:pep8] +basepython = python3 +deps=charm-tools +commands = charm-proof + +[testenv:func-noop] +basepython = python3 +commands = + true + +[testenv:func] +basepython = python3 +commands = functest-run-suite + + +[testenv:func-smoke] +basepython = python3 +commands = functest-run-suite --keep-model --smoke + +[testenv:venv] +commands = {posargs}