osmclient/sol005/vnfd.py

   1 # Copyright 2018 Telefonica
   2 #
   3 # All Rights Reserved.
   4 #
   5 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   6 #    not use this file except in compliance with the License. You may obtain
   7 #    a copy of the License at
   8 #
   9 #         http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #    Unless required by applicable law or agreed to in writing, software
  12 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  13 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  14 #    License for the specific language governing permissions and limitations
  15 #    under the License.
  16
  17 """
  18 OSM vnfd API handling
  19 """
  20
  21 from osmclient.common.exceptions import NotFound
  22 from osmclient.common.exceptions import ClientException
  23 from osmclient.common import utils
  24 import json
  25 import yaml
  26 import magic
  27 from os.path import basename
  28 import logging
  29 import os.path
  30 from urllib.parse import quote
  31 import tarfile
  32 from osm_im.validation import Validation as validation_im
  33 from zipfile import ZipFile
  34
  35
  36 class Vnfd(object):
  37     def __init__(self, http=None, client=None):
  38         self._http = http
  39         self._client = client
  40         self._logger = logging.getLogger("osmclient")
  41         self._apiName = "/vnfpkgm"
  42         self._apiVersion = "/v1"
  43         self._apiResource = "/vnf_packages"
  44         self._apiBase = "{}{}{}".format(
  45             self._apiName, self._apiVersion, self._apiResource
  46         )
  47
  48     def list(self, filter=None):
  49         self._logger.debug("")
  50         self._client.get_token()
  51         filter_string = ""
  52         if filter:
  53             filter_string = "?{}".format(filter)
  54         _, resp = self._http.get2_cmd("{}{}".format(self._apiBase, filter_string))
  55
  56         if resp:
  57             return json.loads(resp)
  58         return list()
  59
  60     def get(self, name):
  61         self._logger.debug("")
  62         self._client.get_token()
  63         if utils.validate_uuid4(name):
  64             for vnfd in self.list():
  65                 if name == vnfd["_id"]:
  66                     return vnfd
  67         else:
  68             for vnfd in self.list():
  69                 if "product-name" in vnfd and name == vnfd["product-name"]:
  70                     return vnfd
  71                 elif "name" in vnfd and name == vnfd["name"]:
  72                     return vnfd
  73         raise NotFound("vnfd {} not found".format(name))
  74
  75     def get_individual(self, name):
  76         self._logger.debug("")
  77         vnfd = self.get(name)
  78         # It is redundant, since the previous one already gets the whole vnfpkginfo
  79         # The only difference is that a different primitive is exercised
  80         try:
  81             _, resp = self._http.get2_cmd("{}/{}".format(self._apiBase, vnfd["_id"]))
  82             if resp:
  83                 return json.loads(resp)
  84         except NotFound:
  85             raise NotFound("vnfd '{}' not found".format(name))
  86         raise NotFound("vnfd '{}' not found".format(name))
  87
  88     def get_thing(self, name, thing, filename):
  89         self._logger.debug("")
  90         vnfd = self.get(name)
  91         headers = self._client._headers
  92         headers["Accept"] = "application/binary"
  93         http_code, resp = self._http.get2_cmd(
  94             "{}/{}/{}".format(self._apiBase, vnfd["_id"], thing)
  95         )
  96
  97         if resp:
  98             return json.loads(resp)
  99
 100     def get_descriptor(self, name, filename):
 101         self._logger.debug("")
 102         self.get_thing(name, "vnfd", filename)
 103
 104     def get_package(self, name, filename):
 105         self._logger.debug("")
 106         self.get_thing(name, "package_content", filename)
 107
 108     def get_artifact(self, name, artifact, filename):
 109         self._logger.debug("")
 110         self.get_thing(name, "artifacts/{}".format(artifact), filename)
 111
 112     def delete(self, name, force=False):
 113         self._logger.debug("")
 114         self._client.get_token()
 115         vnfd = self.get(name)
 116         querystring = ""
 117         if force:
 118             querystring = "?FORCE=True"
 119         http_code, resp = self._http.delete_cmd(
 120             "{}/{}{}".format(self._apiBase, vnfd["_id"], querystring)
 121         )
 122
 123         if http_code == 202:
 124             print("Deletion in progress")
 125         elif http_code == 204:
 126             print("Deleted")
 127         else:
 128             msg = resp or ""
 129             raise ClientException("failed to delete vnfd {} - {}".format(name, msg))
 130
 131     def create(
 132         self,
 133         filename,
 134         overwrite=None,
 135         update_endpoint=None,
 136         skip_charm_build=False,
 137         override_epa=False,
 138         override_nonepa=False,
 139         override_paravirt=False,
 140     ):
 141         self._logger.debug("")
 142         if os.path.isdir(filename):
 143             filename = filename.rstrip("/")
 144             filename = self._client.package_tool.build(
 145                 filename, skip_validation=False, skip_charm_build=skip_charm_build
 146             )
 147
 148             print("Uploading package {}".format(filename))
 149             self.create(
 150                 filename,
 151                 overwrite=overwrite,
 152                 update_endpoint=update_endpoint,
 153                 override_epa=override_epa,
 154                 override_nonepa=override_nonepa,
 155                 override_paravirt=override_paravirt,
 156             )
 157         else:
 158             self._client.get_token()
 159             mime_type = magic.from_file(filename, mime=True)
 160             if mime_type is None:
 161                 raise ClientException(
 162                     "Unexpected MIME type for file {}: MIME type {}".format(
 163                         filename, mime_type
 164                     )
 165                 )
 166             headers = self._client._headers
 167             headers["Content-Filename"] = basename(filename)
 168             if mime_type in ["application/yaml", "text/plain", "application/json"]:
 169                 headers["Content-Type"] = "text/plain"
 170             elif mime_type in ["application/gzip", "application/x-gzip"]:
 171                 headers["Content-Type"] = "application/gzip"
 172             elif mime_type in ["application/zip"]:
 173                 headers["Content-Type"] = "application/zip"
 174             else:
 175                 raise ClientException(
 176                     "Unexpected MIME type for file {}: MIME type {}".format(
 177                         filename, mime_type
 178                     )
 179                 )
 180
 181             special_override_string = ""
 182             if override_epa or override_nonepa or override_paravirt:
 183                 # If override for EPA, non-EPA or paravirt is required, get the descriptor data
 184                 descriptor_data = None
 185                 if mime_type in ["application/yaml", "text/plain", "application/json"]:
 186                     with open(filename) as df:
 187                         descriptor_data = df.read()
 188                 elif mime_type in ["application/gzip", "application/x-gzip"]:
 189                     tar_object = tarfile.open(filename, "r:gz")
 190                     descriptor_list = []
 191                     for member in tar_object:
 192                         if member.isreg():
 193                             if "/" not in os.path.dirname(
 194                                 member.name
 195                             ) and member.name.endswith(".yaml"):
 196                                 descriptor_list.append(member.name)
 197                     if len(descriptor_list) > 1:
 198                         raise ClientException(
 199                             "Found more than one potential descriptor in the tar.gz file"
 200                         )
 201                     elif len(descriptor_list) == 0:
 202                         raise ClientException(
 203                             "No descriptor was found in the tar.gz file"
 204                         )
 205                     with tar_object.extractfile(descriptor_list[0]) as df:
 206                         descriptor_data = df.read()
 207                     tar_object.close()
 208                 elif mime_type in ["application/zip"]:
 209                     package_zip = ZipFile(filename)
 210                     package_files = package_zip.infolist()
 211
 212                     descriptors = []
 213                     if (
 214                         "Definitions" in package_files
 215                         and "TOSCA-Metadata" in package_files
 216                     ):
 217                         descriptors = [
 218                             definition
 219                             for definition in package_files
 220                             if "Definitions/" in definition
 221                             and (
 222                                 definition.endswith(".yaml")
 223                                 or definition.endswith(".yml")
 224                             )
 225                         ]
 226                     else:
 227                         descriptors = [
 228                             definition
 229                             for definition in package_files
 230                             if definition.endswith(".yaml")
 231                             or definition.endswith(".yml")
 232                         ]
 233                     if len(descriptors) < 1:
 234                         raise ClientException(
 235                             "No descriptor found on this package, OSM was expecting at least 1"
 236                         )
 237                     descriptor_data = package_zip.open(descriptors[0])
 238                     package_zip.close()
 239
 240                 if not descriptor_data:
 241                     raise ClientException("Descriptor could not be read")
 242                 desc_type, vnfd = validation_im().yaml_validation(descriptor_data)
 243                 validation_im().pyangbind_validation(desc_type, vnfd)
 244
 245                 vnfd = yaml.safe_load(descriptor_data)
 246                 vcd_list = []
 247                 vdu_list = []
 248                 for k in vnfd:
 249                     # Get only the first descriptor in case there are many in the yaml file
 250                     # k can be vnfd or  etsi-nfv-vnfd:vnfd. This check is skipped
 251                     first_vnfd = vnfd.get(k, {})
 252                     vcd_list = first_vnfd.get("virtual-compute-desc", [])
 253                     vdu_list = first_vnfd.get("vdu", [])
 254                     break
 255
 256                 for vcd_number, vcd in enumerate(vcd_list):
 257                     if override_epa:
 258                         virtual_memory = vcd["virtual-memory"]
 259                         virtual_memory["mempage-size"] = "LARGE"
 260                         virtual_memory["numa-enabled"] = True
 261                         virtual_memory["numa-node-policy"] = {
 262                             "node-cnt": 1,
 263                             "mem-policy": "STRICT",
 264                         }
 265                         virtual_cpu = vcd["virtual-cpu"]
 266                         virtual_cpu["pinning"] = {
 267                             "policy": "static",
 268                             "thread-policy": "PREFER",
 269                         }
 270
 271                         cpu_override_string = (
 272                             "virtual-compute-desc.{}.virtual-cpu={};".format(
 273                                 vcd_number, quote(yaml.safe_dump(virtual_cpu))
 274                             )
 275                         )
 276                         memory_override_string = (
 277                             "virtual-compute-desc.{}.virtual-memory={};".format(
 278                                 vcd_number, quote(yaml.safe_dump(virtual_memory))
 279                             )
 280                         )
 281                         special_override_string = "{}{}{}".format(
 282                             special_override_string,
 283                             cpu_override_string,
 284                             memory_override_string,
 285                         )
 286
 287                     headers["Query-String-Format"] = "yaml"
 288                     if override_nonepa:
 289                         virtual_memory = vcd["virtual-memory"]
 290                         virtual_memory["mempage-size"] = ""
 291                         virtual_memory["numa-enabled"] = ""
 292                         virtual_memory["numa-node-policy"] = {}
 293                         virtual_cpu = vcd["virtual-cpu"]
 294                         virtual_cpu["pinning"] = {}
 295
 296                         cpu_override_string = (
 297                             "virtual-compute-desc.{}.virtual-cpu={};".format(
 298                                 vcd_number, quote(yaml.safe_dump(virtual_cpu))
 299                             )
 300                         )
 301                         memory_override_string = (
 302                             "virtual-compute-desc.{}.virtual-memory={};".format(
 303                                 vcd_number, quote(yaml.safe_dump(virtual_memory))
 304                             )
 305                         )
 306                         special_override_string = "{}{}{}".format(
 307                             special_override_string,
 308                             cpu_override_string,
 309                             memory_override_string,
 310                         )
 311
 312                 if override_paravirt:
 313                     for vdu_number, vdu in enumerate(vdu_list):
 314                         for cpd_number, cpd in enumerate(vdu["int-cpd"]):
 315                             for vnir_number, vnir in enumerate(
 316                                 cpd["virtual-network-interface-requirement"]
 317                             ):
 318                                 special_override_string = (
 319                                     "{}vdu.{}.int-cpd.{}.virtual-network-interface-"
 320                                     "requirement.{}.virtual-interface.type="
 321                                     "PARAVIRT;".format(
 322                                         special_override_string,
 323                                         vdu_number,
 324                                         cpd_number,
 325                                         vnir_number,
 326                                     )
 327                                 )
 328
 329                 special_override_string = special_override_string.rstrip(";")
 330
 331             headers["Content-File-MD5"] = utils.md5(filename)
 332             http_header = [
 333                 "{}: {}".format(key, val) for (key, val) in list(headers.items())
 334             ]
 335
 336             self._http.set_http_header(http_header)
 337             if update_endpoint:
 338                 http_code, resp = self._http.put_cmd(
 339                     endpoint=update_endpoint, filename=filename
 340                 )
 341             else:
 342                 ow_string = ""
 343                 if special_override_string:
 344                     if overwrite:
 345                         overwrite = "{};{}".format(overwrite, special_override_string)
 346                     else:
 347                         overwrite = special_override_string
 348
 349                 if overwrite:
 350                     ow_string = "?{}".format(overwrite)
 351                 self._apiResource = "/vnf_packages_content"
 352                 self._apiBase = "{}{}{}".format(
 353                     self._apiName, self._apiVersion, self._apiResource
 354                 )
 355                 endpoint = "{}{}".format(self._apiBase, ow_string)
 356                 http_code, resp = self._http.post_cmd(
 357                     endpoint=endpoint, filename=filename
 358                 )
 359
 360             if http_code in (200, 201, 202):
 361                 if resp:
 362                     resp = json.loads(resp)
 363                 if not resp or "id" not in resp:
 364                     raise ClientException(
 365                         "unexpected response from server: {}".format(resp)
 366                     )
 367                 print(resp["id"])
 368             elif http_code == 204:
 369                 print("Updated")
 370
 371     def update(self, name, filename):
 372         self._logger.debug("")
 373         self._client.get_token()
 374         vnfd = self.get(name)
 375         endpoint = "{}/{}/package_content".format(self._apiBase, vnfd["_id"])
 376         self.create(filename=filename, update_endpoint=endpoint)