From: Pol González Pacheco Date: Wed, 14 Jun 2023 14:26:15 +0000 (+0200) Subject: Replace pycurl library in osmclient by requests library X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=d4a0c962604935afe9c46024274a0b005e3544d7;p=osm%2Fosmclient.git Replace pycurl library in osmclient by requests library Change-Id: Iab7e5baf25e1917658de627d48e575363787469f Signed-off-by: gonzalezpach --- diff --git a/osmclient/common/http.py b/osmclient/common/http.py index 9207cbc..0e7e78f 100644 --- a/osmclient/common/http.py +++ b/osmclient/common/http.py @@ -14,11 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. -from io import BytesIO -import pycurl +import requests +import urllib3 +from requests.auth import HTTPBasicAuth import json import logging +urllib3.disable_warnings() + class Http(object): def __init__(self, url, user="admin", password="admin"): @@ -31,52 +34,48 @@ class Http(object): def set_http_header(self, header): self._http_header = header - def _get_curl_cmd(self, endpoint): - curl_cmd = pycurl.Curl() - curl_cmd.setopt(pycurl.URL, self._url + endpoint) - curl_cmd.setopt(pycurl.SSL_VERIFYPEER, 0) - curl_cmd.setopt(pycurl.SSL_VERIFYHOST, 0) - curl_cmd.setopt(pycurl.USERPWD, "{}:{}".format(self._user, self._password)) + def _get_requests_cmd(self, endpoint): + requests_cmd = requests.Request() + requests_cmd.url = self._url + endpoint + requests_cmd.auth = HTTPBasicAuth(self._user, self._password) if self._http_header: - curl_cmd.setopt(pycurl.HTTPHEADER, self._http_header) - return curl_cmd + requests_cmd.headers = self._http_header + return requests_cmd def get_cmd(self, endpoint): - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint) - curl_cmd.setopt(pycurl.HTTPGET, 1) - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + session_cmd = requests.Session() + requests_cmd = self._get_requests_cmd(endpoint) + requests_cmd.method = "GET" self._logger.info( "Request METHOD: {} URL: {}".format("GET", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send(requests_cmd, verify=False) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() - if data.getvalue(): - self._logger.debug( - "Response DATA: {}".format(json.loads(data.getvalue().decode())) - ) - return json.loads(data.getvalue().decode()) + data = resp.content + session_cmd.close() + if data: + self._logger.debug("Response DATA: {}".format(json.loads(data.decode()))) + return json.loads(data.decode()) return None def delete_cmd(self, endpoint): - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint) - curl_cmd.setopt(pycurl.CUSTOMREQUEST, "DELETE") - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + session_cmd = requests.Session() + requests_cmd = self._get_requests_cmd(endpoint) + requests_cmd.method = "DELETE" self._logger.info( "Request METHOD: {} URL: {}".format("DELETE", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send(requests_cmd, verify=False) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() - if data.getvalue(): - self._logger.debug( - "Response DATA: {}".format(json.loads(data.getvalue().decode())) - ) - return json.loads(data.getvalue().decode()) + data = resp.content + session_cmd.close() + if data: + self._logger.debug("Response DATA: {}".format(json.loads(data.decode()))) + return json.loads(data.decode()) return None def post_cmd( @@ -85,30 +84,26 @@ class Http(object): postfields_dict=None, formfile=None, ): - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint) - curl_cmd.setopt(pycurl.POST, 1) - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + session_cmd = requests.Session() + requests_cmd = self._get_requests_cmd(endpoint) + requests_cmd.method = "POST" if postfields_dict is not None: - jsondata = json.dumps(postfields_dict) - curl_cmd.setopt(pycurl.POSTFIELDS, jsondata) + requests_cmd.json = json.dumps(postfields_dict) if formfile is not None: - curl_cmd.setopt( - pycurl.HTTPPOST, [((formfile[0], (pycurl.FORM_FILE, formfile[1])))] - ) + requests_cmd.files = {formfile[0]: formfile[1]} self._logger.info( "Request METHOD: {} URL: {}".format("POST", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send(requests_cmd, verify=False) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() - if data.getvalue(): - self._logger.debug( - "Response DATA: {}".format(json.loads(data.getvalue().decode())) - ) - return json.loads(data.getvalue().decode()) + data = resp.content + session_cmd.close() + if data: + self._logger.debug("Response DATA: {}".format(json.loads(data.decode()))) + return json.loads(data.decode()) return None diff --git a/osmclient/sol005/client.py b/osmclient/sol005/client.py index 0cbc80a..b05dbcc 100644 --- a/osmclient/sol005/client.py +++ b/osmclient/sol005/client.py @@ -77,10 +77,7 @@ class Client(object): ) self._headers["Accept"] = "application/json" self._headers["Content-Type"] = "application/yaml" - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(self._headers.items()) - ] - self._http_client.set_http_header(http_header) + self._http_client.set_http_header(self._headers) self.vnfd = vnfd.Vnfd(self._http_client, client=self) self.nsd = nsd.Nsd(self._http_client, client=self) @@ -137,11 +134,7 @@ class Client(object): if self._token is not None: self._headers["Authorization"] = "Bearer {}".format(self._token) - http_header = [ - "{}: {}".format(key, val) - for (key, val) in list(self._headers.items()) - ] - self._http_client.set_http_header(http_header) + self._http_client.set_http_header(self._headers) return token def get_version(self): diff --git a/osmclient/sol005/http.py b/osmclient/sol005/http.py index 4234fc8..8dd87e7 100644 --- a/osmclient/sol005/http.py +++ b/osmclient/sol005/http.py @@ -15,13 +15,17 @@ # under the License. import copy -from io import BytesIO import json import logging +import requests +import http.client as http_client +import urllib3 from osmclient.common import http from osmclient.common.exceptions import OsmHttpException, NotFound -import pycurl + + +urllib3.disable_warnings() class Http(http.Http): @@ -58,42 +62,50 @@ class Http(http.Http): endpoint = "?".join([endpoint, self._default_query_admin]) return endpoint - def _get_curl_cmd(self, endpoint, skip_query_admin=False): + def _get_requests_cmd(self, endpoint, skip_query_admin=False): self._logger.debug("") - curl_cmd = pycurl.Curl() + requests_cmd = requests.Request() if self._logger.getEffectiveLevel() == logging.DEBUG: - curl_cmd.setopt(pycurl.VERBOSE, True) + http_client.HTTPConnection.debuglevel = 1 + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.DEBUG) + requests_log.propagate = True + else: + http_client.HTTPConnection.debuglevel = 0 + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.NOTSET) + requests_log.propagate = True + if not skip_query_admin: endpoint = self._complete_endpoint(endpoint) - curl_cmd.setopt(pycurl.CONNECTTIMEOUT, self.CONNECT_TIMEOUT) - curl_cmd.setopt(pycurl.URL, self._url + endpoint) - curl_cmd.setopt(pycurl.SSL_VERIFYPEER, 0) - curl_cmd.setopt(pycurl.SSL_VERIFYHOST, 0) + requests_cmd.url = self._url + endpoint if self._http_header: - curl_cmd.setopt(pycurl.HTTPHEADER, self._http_header) - return curl_cmd + requests_cmd.headers = self._http_header + return requests_cmd def delete_cmd(self, endpoint, skip_query_admin=False): + session_cmd = requests.Session() self._logger.debug("") - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint, skip_query_admin) - curl_cmd.setopt(pycurl.CUSTOMREQUEST, "DELETE") - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + requests_cmd = self._get_requests_cmd(endpoint, skip_query_admin) + requests_cmd.method = "DELETE" self._logger.info( "Request METHOD: {} URL: {}".format("DELETE", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send( + requests_cmd, verify=False, timeout=self.CONNECT_TIMEOUT + ) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() + data = resp.content + session_cmd.close() self.check_http_response(http_code, data) # TODO 202 accepted should be returned somehow - if data.getvalue(): - data_text = data.getvalue().decode() + if data: + data_text = data.decode() self._logger.verbose("Response DATA: {}".format(data_text)) return http_code, data_text - else: - return http_code, None + return http_code, None def send_cmd( self, @@ -105,15 +117,15 @@ class Http(http.Http): patch_method=False, skip_query_admin=False, ): + session_cmd = requests.Session() self._logger.debug("") - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint, skip_query_admin) + requests_cmd = self._get_requests_cmd(endpoint, skip_query_admin) if put_method: - curl_cmd.setopt(pycurl.CUSTOMREQUEST, "PUT") + requests_cmd.method = "PUT" elif patch_method: - curl_cmd.setopt(pycurl.CUSTOMREQUEST, "PATCH") - curl_cmd.setopt(pycurl.POST, 1) - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + requests_cmd.method = "PATCH" + else: + requests_cmd.method = "POST" if postfields_dict is not None: jsondata = json.dumps(postfields_dict) @@ -123,17 +135,15 @@ class Http(http.Http): jsondata_log = json.dumps(postfields_dict_copy) else: jsondata_log = jsondata + requests_cmd.json = postfields_dict self._logger.verbose("Request POSTFIELDS: {}".format(jsondata_log)) - curl_cmd.setopt(pycurl.POSTFIELDS, jsondata) elif formfile is not None: - curl_cmd.setopt( - pycurl.HTTPPOST, [((formfile[0], (pycurl.FORM_FILE, formfile[1])))] - ) + requests_cmd.files = {formfile[0]: formfile[1]} elif filename is not None: with open(filename, "rb") as stream: postdata = stream.read() self._logger.verbose("Request POSTFIELDS: Binary content") - curl_cmd.setopt(pycurl.POSTFIELDS, postdata) + requests_cmd.data = postdata if put_method: self._logger.info( @@ -147,17 +157,20 @@ class Http(http.Http): self._logger.info( "Request METHOD: {} URL: {}".format("POST", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send( + requests_cmd, verify=False, timeout=self.CONNECT_TIMEOUT + ) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() + data = resp.content self.check_http_response(http_code, data) - if data.getvalue(): - data_text = data.getvalue().decode() + session_cmd.close() + if data: + data_text = data.decode() self._logger.verbose("Response DATA: {}".format(data_text)) return http_code, data_text - else: - return http_code, None + return http_code, None def post_cmd( self, @@ -217,21 +230,24 @@ class Http(http.Http): ) def get2_cmd(self, endpoint, skip_query_admin=False): + session_cmd = requests.Session() self._logger.debug("") - data = BytesIO() - curl_cmd = self._get_curl_cmd(endpoint, skip_query_admin) - curl_cmd.setopt(pycurl.HTTPGET, 1) - curl_cmd.setopt(pycurl.WRITEFUNCTION, data.write) + requests_cmd = self._get_requests_cmd(endpoint, skip_query_admin) + requests_cmd.method = "GET" self._logger.info( "Request METHOD: {} URL: {}".format("GET", self._url + endpoint) ) - curl_cmd.perform() - http_code = curl_cmd.getinfo(pycurl.HTTP_CODE) + requests_cmd = requests_cmd.prepare() + resp = session_cmd.send( + requests_cmd, verify=False, timeout=self.CONNECT_TIMEOUT + ) + http_code = resp.status_code self._logger.info("Response HTTPCODE: {}".format(http_code)) - curl_cmd.close() + data = resp.content + session_cmd.close() self.check_http_response(http_code, data) - if data.getvalue(): - data_text = data.getvalue().decode() + if data: + data_text = data.decode() self._logger.verbose("Response DATA: {}".format(data_text)) return http_code, data_text return http_code, None @@ -239,8 +255,8 @@ class Http(http.Http): def check_http_response(self, http_code, data): if http_code >= 300: resp = "" - if data.getvalue(): - data_text = data.getvalue().decode() + if data: + data_text = data.decode() self._logger.verbose( "Response {} DATA: {}".format(http_code, data_text) ) diff --git a/osmclient/sol005/ns.py b/osmclient/sol005/ns.py index 00d68d6..c84d090 100644 --- a/osmclient/sol005/ns.py +++ b/osmclient/sol005/ns.py @@ -278,10 +278,7 @@ class Ns(object): ) headers = self._client._headers headers["Content-Type"] = "application/yaml" - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) http_code, resp = self._http.post_cmd( endpoint=self._apiBase, postfields_dict=ns ) diff --git a/osmclient/sol005/nsd.py b/osmclient/sol005/nsd.py index 5ccbe78..2fb5859 100644 --- a/osmclient/sol005/nsd.py +++ b/osmclient/sol005/nsd.py @@ -161,10 +161,7 @@ class Nsd(object): ) ) headers["Content-File-MD5"] = utils.md5(filename) - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) if update_endpoint: http_code, resp = self._http.put_cmd( endpoint=update_endpoint, filename=filename diff --git a/osmclient/sol005/nsi.py b/osmclient/sol005/nsi.py index 6efce4b..b9d5e4b 100644 --- a/osmclient/sol005/nsi.py +++ b/osmclient/sol005/nsi.py @@ -260,10 +260,7 @@ class Nsi(object): ) headers = self._client._headers headers["Content-Type"] = "application/yaml" - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) http_code, resp = self._http.post_cmd( endpoint=self._apiBase, postfields_dict=nsi ) diff --git a/osmclient/sol005/nst.py b/osmclient/sol005/nst.py index 19092b2..989e0bd 100644 --- a/osmclient/sol005/nst.py +++ b/osmclient/sol005/nst.py @@ -192,10 +192,7 @@ class Nst(object): ) ) headers["Content-File-MD5"] = utils.md5(filename) - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) if update_endpoint: http_code, resp = self._http.put_cmd( endpoint=update_endpoint, filename=filename diff --git a/osmclient/sol005/package.py b/osmclient/sol005/package.py index 79125d8..a083300 100644 --- a/osmclient/sol005/package.py +++ b/osmclient/sol005/package.py @@ -105,10 +105,7 @@ class Package(object): # file_size = stat(filename).st_size # headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) headers["Content-File-MD5"] = utils.md5(filename) - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) # print('HTTP CODE: {}'.format(http_code)) # print('RESP: {}'.format(resp)) diff --git a/osmclient/sol005/pdud.py b/osmclient/sol005/pdud.py index e47dadd..942516c 100644 --- a/osmclient/sol005/pdud.py +++ b/osmclient/sol005/pdud.py @@ -118,10 +118,7 @@ class Pdu(object): self._client.get_token() headers = self._client._headers headers["Content-Type"] = "application/yaml" - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - self._http.set_http_header(http_header) + self._http.set_http_header(headers) if update_endpoint: http_code, resp = self._http.patch_cmd( endpoint=update_endpoint, postfields_dict=pdu diff --git a/osmclient/sol005/vnfd.py b/osmclient/sol005/vnfd.py index 0a2b772..3c09707 100644 --- a/osmclient/sol005/vnfd.py +++ b/osmclient/sol005/vnfd.py @@ -329,11 +329,7 @@ class Vnfd(object): special_override_string = special_override_string.rstrip(";") headers["Content-File-MD5"] = utils.md5(filename) - http_header = [ - "{}: {}".format(key, val) for (key, val) in list(headers.items()) - ] - - self._http.set_http_header(http_header) + self._http.set_http_header(headers) if update_endpoint: http_code, resp = self._http.put_cmd( endpoint=update_endpoint, filename=filename diff --git a/requirements.in b/requirements.in index e73bacc..02b5ed3 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,6 @@ Click jinja2 packaging prettytable -pycurl python-magic pyyaml==5.4.1 requests diff --git a/requirements.txt b/requirements.txt index d326224..421e66d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ idna==3.4 # via requests jinja2==3.1.2 # via -r requirements.in -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 packaging==23.1 # via -r requirements.in