From d5916fd2af450e9e146100a65e632e3cc0400748 Mon Sep 17 00:00:00 2001 From: calvinosanch Date: Thu, 9 Jan 2020 17:19:53 +0100 Subject: [PATCH] Ubuntu18 + pip upgraded + aiohttp Change-Id: If7e62cfdf00f01af4e4f8a8cd1afee311c8ed4aa Signed-off-by: calvinosanch --- Dockerfile.local | 8 +- debian/python3-osm-lcm.postinst | 4 +- osm_lcm/ROclient.py | 156 +++++++++++++++++--------------- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 90 insertions(+), 82 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 9b25519..75bdd90 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:16.04 +FROM ubuntu:18.04 # Set the working directory to /app WORKDIR /app/LCM -RUN apt-get update && apt-get install -y curl xz-utils \ +RUN apt-get update && apt-get install -y curl xz-utils gnupg2 \ && apt-get update && apt-get install -y apt-transport-https \ && curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ @@ -34,9 +34,9 @@ RUN curl -L https://launchpad.net/juju/2.7/2.7.0/+download/juju-2.7.0-k8s.tar.xz && mv juju /usr/local/bin/juju RUN apt-get update && apt-get install -y git tox python3 \ - python3-pip python3-aiohttp python3-jinja2 \ + python3-pip python3-jinja2 \ && python3 -m pip install pip --upgrade \ - && python3 -m pip install -U 'PyYAML==3.*' 'aiohttp==0.20.2' flake8 + && python3 -m pip install -U 'PyYAML' 'aiohttp>=2.3.10' flake8 RUN git -C /app clone https://osm.etsi.org/gerrit/osm/N2VC.git \ && python3 -m pip install -e /app/N2VC \ diff --git a/debian/python3-osm-lcm.postinst b/debian/python3-osm-lcm.postinst index 7ee6c6c..f66671f 100755 --- a/debian/python3-osm-lcm.postinst +++ b/debian/python3-osm-lcm.postinst @@ -20,8 +20,8 @@ echo "POST INSTALL OSM-LCM" # Currently it is not needed pip3 installation # echo "Installing python dependencies via pip..." -# pip3 install pip==9.0.3 -# pip3 install --user aiokafka +# python3 -m pip install -U pip +# python3 -m pip install --user aiokafka #Creation of log folder mkdir -p /var/log/osm diff --git a/osm_lcm/ROclient.py b/osm_lcm/ROclient.py index 54dbbf1..4d5600b 100644 --- a/osm_lcm/ROclient.py +++ b/osm_lcm/ROclient.py @@ -441,16 +441,16 @@ class ROClient: elif item_id_name and item_id_name.startswith("'") and item_id_name.endswith("'"): item_id_name = item_id_name[1:-1] self.logger.debug("RO GET %s", url) - with aiohttp.Timeout(self.timeout_short): - async with session.get(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status == 404: # NOT_FOUND - raise ROClientException("No {} found with id '{}'".format(item[:-1], item_id_name), - http_code=404) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) - content = self._parse_yaml(response_text, response=True) + # timeout = aiohttp.ClientTimeout(total=self.timeout_short) + async with session.get(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status == 404: # NOT_FOUND + raise ROClientException("No {} found with id '{}'".format(item[:-1], item_id_name), + http_code=404) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + content = self._parse_yaml(response_text, response=True) if item_id: return item_id @@ -491,12 +491,12 @@ class ROClient: if extra_item_id: url += "/" + extra_item_id self.logger.debug("GET %s", url) - with aiohttp.Timeout(self.timeout_short): - async with session.get(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_short) + async with session.get(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) return self._parse_yaml(response_text, response=True) @@ -550,12 +550,12 @@ class ROClient: url = "{}{apiver}{tenant}/{item}{id}{action}".format(self.endpoint_url, apiver=api_version_text, tenant=tenant_text, item=item, id=uuid, action=action) self.logger.debug("RO POST %s %s", url, payload_req) - with aiohttp.Timeout(self.timeout_large): - async with session.post(url, headers=self.headers_req, data=payload_req) as response: - response_text = await response.read() - self.logger.debug("POST {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_large) + async with session.post(url, headers=self.headers_req, data=payload_req) as response: + response_text = await response.read() + self.logger.debug("POST {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) return self._parse_yaml(response_text, response=True) @@ -579,12 +579,13 @@ class ROClient: url = "{}{}/{}/{}".format(self.endpoint_url, tenant_text, item, uuid) self.logger.debug("DELETE %s", url) - with aiohttp.Timeout(self.timeout_short): - async with session.delete(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("DELETE {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_short) + async with session.delete(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("DELETE {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + return self._parse_yaml(response_text, response=True) async def _list_item(self, session, item, all_tenants=False, filter_dict=None): @@ -604,12 +605,13 @@ class ROClient: url += separator + quote(str(k)) + "=" + quote(str(filter_dict[k])) separator = "&" self.logger.debug("RO GET %s", url) - with aiohttp.Timeout(self.timeout_short): - async with session.get(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_short) + async with session.get(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + return self._parse_yaml(response_text, response=True) async def _edit_item(self, session, item, item_id, descriptor, all_tenants=False): @@ -627,12 +629,13 @@ class ROClient: # print payload_req url = "{}{}/{}/{}".format(self.endpoint_url, tenant_text, item, item_id) self.logger.debug("RO PUT %s %s", url, payload_req) - with aiohttp.Timeout(self.timeout_large): - async with session.put(url, headers=self.headers_req, data=payload_req) as response: - response_text = await response.read() - self.logger.debug("PUT {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_large) + async with session.put(url, headers=self.headers_req, data=payload_req) as response: + response_text = await response.read() + self.logger.debug("PUT {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + return self._parse_yaml(response_text, response=True) async def get_version(self): @@ -641,21 +644,22 @@ class ROClient: :return: a list with integers ["major", "minor", "release"]. Raises ROClientException on Error, """ try: - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: url = "{}/version".format(self.endpoint_url) self.logger.debug("RO GET %s", url) - with aiohttp.Timeout(self.timeout_short): - async with session.get(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_short) + async with session.get(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("GET {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + for word in str(response_text).split(" "): if "." in word: version_text, _, _ = word.partition("-") return version_text raise ROClientException("Got invalid version text: '{}'".format(response_text), http_code=500) - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -676,7 +680,7 @@ class ROClient: raise ROClientException("Invalid item {}".format(item)) if item == 'tenant': all_tenants = None - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: content = await self._list_item(session, self.client_to_RO[item], all_tenants=all_tenants, filter_dict=filter_by) if isinstance(content, dict): @@ -687,7 +691,7 @@ class ROClient: else: raise ROClientException("Output not a list neither dict with len equal 1", http_code=500) return content - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -713,11 +717,11 @@ class ROClient: elif item == 'vim_account': all_tenants = False - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: content = await self._get_item(session, self.client_to_RO[item], item_id_name, extra_item=extra_item, extra_item_id=extra_item_id, all_tenants=all_tenants) return remove_envelop(item, content) - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -736,7 +740,7 @@ class ROClient: if item in ('tenant', 'vim', 'wim'): all_tenants = None - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: result = await self._del_item(session, self.client_to_RO[item], item_id_name, all_tenants=all_tenants) # in case of ns delete, get the action_id embeded in text if item == "ns" and result.get("result"): @@ -745,7 +749,7 @@ class ROClient: if action_id: result["action_id"] = action_id return result - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -781,7 +785,7 @@ class ROClient: create_desc = self._create_envelop(item, desc) - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: _all_tenants = all_tenants if item == 'vim': _all_tenants = True @@ -793,7 +797,7 @@ class ROClient: outdata = await self._edit_item(session, self.client_to_RO[item], item_id, create_desc, all_tenants=_all_tenants) return remove_envelop(item, outdata) - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -834,11 +838,11 @@ class ROClient: create_desc = self._create_envelop(item, desc) - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: outdata = await self._create_item(session, self.client_to_RO[item], create_desc, all_tenants=all_tenants) return remove_envelop(item, outdata) - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -883,7 +887,7 @@ class ROClient: # create_desc = self._create_envelop(item, desc) create_desc = desc - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: _all_tenants = all_tenants if item == 'vim': _all_tenants = True @@ -893,7 +897,7 @@ class ROClient: item_id_name=item_id_name, # item_id_name=item_id action=action, all_tenants=_all_tenants) return remove_envelop(item, outdata) - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -933,7 +937,7 @@ class ROClient: format(item)) create_desc = self._create_envelop(item, desc) payload_req = yaml.safe_dump(create_desc) - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: # check that exist item_id = await self._get_item_uuid(session, self.client_to_RO[item], item_id_name, all_tenants=True) await self._get_tenant(session) @@ -941,17 +945,17 @@ class ROClient: url = "{}/{tenant}/{item}/{item_id}".format(self.endpoint_url, tenant=self.tenant, item=self.client_to_RO[item], item_id=item_id) self.logger.debug("RO POST %s %s", url, payload_req) - with aiohttp.Timeout(self.timeout_large): - async with session.post(url, headers=self.headers_req, data=payload_req) as response: - response_text = await response.read() - self.logger.debug("POST {} [{}] {}".format(url, response.status, response_text[:100])) - if response.status >= 300: - raise ROClientException(response_text, http_code=response.status) + # timeout = aiohttp.ClientTimeout(total=self.timeout_large) + async with session.post(url, headers=self.headers_req, data=payload_req) as response: + response_text = await response.read() + self.logger.debug("POST {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) response_desc = self._parse_yaml(response_text, response=True) desc = remove_envelop(item, response_desc) return desc - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) @@ -959,7 +963,7 @@ class ROClient: async def detach(self, item, item_id_name=None): # TODO replace the code with delete_item(vim_account,...) try: - with aiohttp.ClientSession(loop=self.loop) as session: + async with aiohttp.ClientSession(loop=self.loop) as session: # check that exist item_id = await self._get_item_uuid(session, self.client_to_RO[item], item_id_name, all_tenants=False) tenant = await self._get_tenant(session) @@ -967,17 +971,21 @@ class ROClient: url = "{}/{tenant}/{item}/{datacenter}".format(self.endpoint_url, tenant=tenant, item=self.client_to_RO[item], datacenter=item_id) self.logger.debug("RO DELETE %s", url) - with aiohttp.Timeout(self.timeout_large): - async with session.delete(url, headers=self.headers_req) as response: - response_text = await response.read() - self.logger.debug("DELETE {} [{}] {}".format(url, response.status, response_text[:100])) + + # timeout = aiohttp.ClientTimeout(total=self.timeout_large) + async with session.delete(url, headers=self.headers_req) as response: + response_text = await response.read() + self.logger.debug("DELETE {} [{}] {}".format(url, response.status, response_text[:100])) + if response.status >= 300: + raise ROClientException(response_text, http_code=response.status) + if response.status >= 300: raise ROClientException(response_text, http_code=response.status) response_desc = self._parse_yaml(response_text, response=True) desc = remove_envelop(item, response_desc) return desc - except aiohttp.errors.ClientOSError as e: + except aiohttp.ClientOSError as e: raise ROClientException(e, http_code=504) except asyncio.TimeoutError: raise ROClientException("Timeout", http_code=504) diff --git a/requirements.txt b/requirements.txt index ea2d1c2..a1b035b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ ## pyyaml -aiohttp==0.20.2 +aiohttp>=2.3.10 jinja2 git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common git+https://osm.etsi.org/gerrit/osm/N2VC.git#egg=n2vc diff --git a/setup.py b/setup.py index b4d614a..dddfc02 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( install_requires=[ # 'pymongo', 'PyYAML', - 'aiohttp==0.20.2', + 'aiohttp==3.*', 'osm-common', 'n2vc', 'jinja2', -- 2.17.1