From: tierno Date: Mon, 3 Dec 2018 16:35:42 +0000 (+0000) Subject: bug 605 DescriptorTopic upload_content copying to temporal folder for rollback if... X-Git-Tag: v5.0.3~15 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=commitdiff_plain;h=f717cbe40358a8a8246d7936064cf235e2ff7f79 bug 605 DescriptorTopic upload_content copying to temporal folder for rollback if needed Change-Id: Ia3640e7900cd9a1aea3756e18db87a2b754d1853 Signed-off-by: tierno --- diff --git a/osm_nbi/descriptor_topics.py b/osm_nbi/descriptor_topics.py index e9339e0..5eeab92 100644 --- a/osm_nbi/descriptor_topics.py +++ b/osm_nbi/descriptor_topics.py @@ -85,6 +85,7 @@ class DescriptorTopic(BaseTopic): return v = self.db.del_one(self.topic, {"_id": _id}) self.fs.file_delete(_id, ignore_non_exist=True) + self.fs.file_delete(_id + "_", ignore_non_exist=True) # remove temp folder self._send_msg("delete", {"_id": _id}) return v @@ -185,18 +186,19 @@ class DescriptorTopic(BaseTopic): total = int(content_range[3]) else: start = 0 + temp_folder = _id + "_" # all the content is upload here and if ok, it is rename from id_ to is folder if start: - if not self.fs.file_exists(_id, 'dir'): + if not self.fs.file_exists(temp_folder, 'dir'): raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND) else: - self.fs.file_delete(_id, ignore_non_exist=True) - self.fs.mkdir(_id) + self.fs.file_delete(temp_folder, ignore_non_exist=True) + self.fs.mkdir(temp_folder) storage = self.fs.get_params() storage["folder"] = _id - file_path = (_id, filename) + file_path = (temp_folder, filename) if self.fs.file_exists(file_path, 'file'): file_size = self.fs.file_size(file_path) else: @@ -256,8 +258,8 @@ class DescriptorTopic(BaseTopic): raise EngineException("Not found any descriptor file at package descriptor tar.gz") storage["descriptor"] = descriptor_file_name storage["zipfile"] = filename - self.fs.file_extract(tar, _id) - with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file: + self.fs.file_extract(tar, temp_folder) + with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file: content = descriptor_file.read() else: content = file_pkg.read() @@ -285,6 +287,7 @@ class DescriptorTopic(BaseTopic): deep_update_rfc7396(current_desc, indata) self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force) self.db.replace(self.topic, _id, current_desc) + self.fs.dir_rename(temp_folder, _id) indata["_id"] = _id self._send_msg("created", indata) diff --git a/osm_nbi/engine.py b/osm_nbi/engine.py index 1eff5e1..14dab56 100644 --- a/osm_nbi/engine.py +++ b/osm_nbi/engine.py @@ -225,13 +225,6 @@ class Engine(object): raise EngineException("Unknown topic {}!!!".format(topic), HTTPStatus.INTERNAL_SERVER_ERROR) return self.map_topic[topic].edit(session, _id, indata, kwargs, force) - def prune(self): - """ - Prune database not needed content - :return: None - """ - return self.db.del_list("nsrs", {"_admin.to_delete": True}) - def create_admin(self): """ Creates a new user admin/admin into database if database is empty. Useful for initialization diff --git a/osm_nbi/html_public/version b/osm_nbi/html_public/version index 0f7d7fd..279998e 100644 --- a/osm_nbi/html_public/version +++ b/osm_nbi/html_public/version @@ -1,2 +1,2 @@ -0.1.29 -2018-11-20 +0.1.30 +2018-12-05 diff --git a/osm_nbi/nbi.py b/osm_nbi/nbi.py index caf4c2d..2dfeb22 100644 --- a/osm_nbi/nbi.py +++ b/osm_nbi/nbi.py @@ -444,7 +444,8 @@ class Server(object): elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept: pass - else: + # if there is not any valid accept, raise an error. But if response is already an error, format in yaml + elif cherrypy.response.status >= 400: raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value, "Only 'Accept' of type 'application/json' or 'application/yaml' " "for output format are available") @@ -544,7 +545,7 @@ class Server(object): def test(self, *args, **kwargs): thread_info = None if args and args[0] == "help": - return "
\ninit\nfile/  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
+            return "
\ninit\nfile/  download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"\
                    "sleep/
" elif args and args[0] == "init": @@ -565,9 +566,16 @@ class Server(object): return f elif len(args) == 2 and args[0] == "db-clear": - return self.engine.db.del_list(args[1], kwargs) - elif args and args[0] == "prune": - return self.engine.prune() + deleted_info = self.engine.db.del_list(args[1], kwargs) + return "{} {} deleted\n".format(deleted_info["deleted"], args[1]) + elif len(args) and args[0] == "fs-clear": + if len(args) >= 2: + folders = (args[1],) + else: + folders = self.engine.fs.dir_ls(".") + for folder in folders: + self.engine.fs.file_delete(folder) + return ",".join(folders) + " folders deleted\n" elif args and args[0] == "login": if not cherrypy.request.headers.get("Authorization"): cherrypy.response.headers["WWW-Authenticate"] = 'Basic realm="Access to OSM site", charset="UTF-8"' @@ -614,7 +622,7 @@ class Server(object): " session: {}\n".format(cherrypy.session) + " cookie: {}\n".format(cherrypy.request.cookie) + " method: {}\n".format(cherrypy.request.method) + - " session: {}\n".format(cherrypy.session.get('fieldname')) + + " session: {}\n".format(cherrypy.session.get('fieldname')) + " body:\n") return_text += " length: {}\n".format(cherrypy.request.body.length) if cherrypy.request.body.length: diff --git a/osm_nbi/tests/clear-all.sh b/osm_nbi/tests/clear-all.sh index dc9dca7..9b6b8c6 100755 --- a/osm_nbi/tests/clear-all.sh +++ b/osm_nbi/tests/clear-all.sh @@ -83,11 +83,11 @@ fi for item in vim_accounts sdns nsrs vnfrs nslcmops nsds vnfds projects pdus nsts nsis nsilcmops # vims do curl --insecure ${OSMNBI_URL}/test/db-clear/${item} - echo " ${item}" done + curl --insecure ${OSMNBI_URL}/test/fs-clear if [ -n "$OSMNBI_COMPLETELY" ] ; then - curl --insecure ${OSMNBI_URL}/test/db-clear/users && echo " ${item}" - curl --insecure ${OSMNBI_URL}/test/db-clear/version && echo " ${item}" + curl --insecure ${OSMNBI_URL}/test/db-clear/users + curl --insecure ${OSMNBI_URL}/test/db-clear/admin else # delete all users except admin curl --insecure ${OSMNBI_URL}/test/db-clear/users?username.ne=admin diff --git a/osm_nbi/tests/test.py b/osm_nbi/tests/test.py index 8621d6b..389d14b 100755 --- a/osm_nbi/tests/test.py +++ b/osm_nbi/tests/test.py @@ -72,28 +72,19 @@ def usage(): r_header_json = {"Content-type": "application/json"} -headers_json = { - "Content-type": "application/json", - "Accept": "application/json", -} +headers_json = {"Content-type": "application/json", "Accept": "application/json"} r_header_yaml = {"Content-type": "application/yaml"} -headers_yaml = { - "Content-type": "application/yaml", - "Accept": "application/yaml", -} +headers_yaml = {"Content-type": "application/yaml", "Accept": "application/yaml"} r_header_text = {"Content-type": "text/plain"} r_header_octect = {"Content-type": "application/octet-stream"} -headers_text = { - "Accept": "text/plain", -} +headers_text = {"Accept": "text/plain,application/yaml"} r_header_zip = {"Content-type": "application/zip"} -headers_zip = { - "Accept": "application/zip", -} -headers_zip_yaml = { - "Accept": "application/yaml", "Content-type": "application/zip" -} - +headers_zip = {"Accept": "application/zip,application/yaml"} +headers_zip_yaml = {"Accept": "application/yaml", "Content-type": "application/zip"} +r_headers_yaml_location_vnfd = {"Location": "/vnfpkgm/v1/vnf_packages_content/", "Content-Type": "application/yaml"} +r_headers_yaml_location_nsd = {"Location": "/nsd/v1/ns_descriptors_content/", "Content-Type": "application/yaml"} +r_headers_yaml_location_nst = {"Location": "/nst/v1/netslice_templates_content", "Content-Type": "application/yaml"} +r_headers_yaml_location_nslcmop = {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"} # test ones authorized test_authorized_list = ( @@ -712,7 +703,7 @@ class TestDeploy: # vnfd CREATE AND UPLOAD in one step: engine.test("Onboard VNFD in one step", "POST", "/vnfpkgm/v1/vnf_packages_content" + self.qforce, headers, "@b" + vnfd_filename_path, 201, - {"Location": "/vnfpkgm/v1/vnf_packages_content/", "Content-Type": "application/yaml"}, + r_headers_yaml_location_vnfd, "yaml") self.vnfds_id.append(engine.last_id) else: @@ -754,7 +745,7 @@ class TestDeploy: # nsd CREATE AND UPLOAD in one step: engine.test("Onboard NSD in one step", "POST", "/nsd/v1/ns_descriptors_content" + self.qforce, headers, "@b" + nsd_filename_path, 201, - {"Location": "/nsd/v1/ns_descriptors_content/", "Content-Type": "application/yaml"}, yaml) + r_headers_yaml_location_nsd, yaml) self.nsd_id = engine.last_id else: # nsd CREATE AND UPLOAD ZIP @@ -792,7 +783,7 @@ class TestDeploy: self.ns_id = engine.last_id engine.test("Instantiate NS step 2", "POST", "/nslcm/v1/ns_instances/{}/instantiate".format(self.ns_id), headers_yaml, ns_data_text, - 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop_id = engine.last_id if test_osm: @@ -804,7 +795,7 @@ class TestDeploy: # remove deployment if test_osm: engine.test("Terminate NS", "POST", "/nslcm/v1/ns_instances/{}/terminate".format(self.ns_id), headers_yaml, - None, 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + None, 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop2_id = engine.last_id # Wait until status is Ok engine.wait_operation_ready("ns", nslcmop2_id, timeout_deploy) @@ -1015,7 +1006,7 @@ class TestDeployHackfestCirrosScaling(TestDeploy): for i in range(0, 2): engine.test("Execute scale action over NS", "POST", "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload, - 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop2_scale_out = engine.last_id engine.wait_operation_ready("ns", nslcmop2_scale_out, timeout_deploy) if manual_check: @@ -1028,7 +1019,7 @@ class TestDeployHackfestCirrosScaling(TestDeploy): for i in range(0, 2): engine.test("Execute scale IN action over NS", "POST", "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload, - 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop2_scale_in = engine.last_id engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy) if manual_check: @@ -1038,7 +1029,7 @@ class TestDeployHackfestCirrosScaling(TestDeploy): # perform scale in that must fail as reached limit engine.test("Execute scale IN out of limit action over NS", "POST", "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload, - 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop2_scale_in = engine.last_id engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy, expected_fail=True) @@ -1233,7 +1224,7 @@ class TestDeployHackfest3Charmed(TestDeploy): payload = '{member_vnf_index: "2", primitive: touch, primitive_params: { filename: /home/ubuntu/OSMTESTNBI }}' engine.test("Exec service primitive over NS", "POST", "/nslcm/v1/ns_instances/{}/action".format(self.ns_id), headers_yaml, payload, - 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nslcmop, "yaml") nslcmop2_action = engine.last_id # Wait until status is Ok engine.wait_operation_ready("ns", nslcmop2_action, timeout_deploy) @@ -1251,7 +1242,7 @@ class TestDeployHackfest3Charmed(TestDeploy): # '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}' # engine.test("Execute scale action over NS", "POST", # "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload, - # 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + # 201, r_headers_yaml_location_nslcmop, "yaml") # nslcmop2_scale_out = engine.last_id # engine.wait_operation_ready("ns", nslcmop2_scale_out, timeout_deploy) # if manual_check: @@ -1263,7 +1254,7 @@ class TestDeployHackfest3Charmed(TestDeploy): # '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}' # engine.test("Execute scale action over NS", "POST", # "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload, - # 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + # 201, r_headers_yaml_location_nslcmop, "yaml") # nslcmop2_scale_in = engine.last_id # engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy) # if manual_check: @@ -1631,6 +1622,31 @@ class TestDescriptors: self.descriptor_url = "https://osm-download.etsi.org/ftp/osm-3.0-three/2nd-hackfest/packages/" self.vnfd_id = None self.nsd_id = None + self.vnfd_empty = """vnfd:vnfd-catalog: + vnfd: + - name: prova + short-name: prova + id: prova + """ + self.vnfd_prova = """vnfd:vnfd-catalog: + vnfd: + - connection-point: + - name: cp_0h8m + type: VPORT + id: prova + name: prova + short-name: prova + vdu: + - id: vdu_z4bm + image: ubuntu + interface: + - external-connection-point-ref: cp_0h8m + name: eth0 + virtual-interface: + type: VIRTIO + name: vdu_z4bm + version: '1.0' + """ def run(self, engine, test_osm, manual_check, test_params=None): engine.set_test_name("Descriptors") @@ -1653,20 +1669,29 @@ class TestDescriptors: vnfd_filename_path = temp_dir + self.vnfd_filename nsd_filename_path = temp_dir + self.nsd_filename - # vnfd CREATE AND UPLOAD in one step: - engine.test("Onboard VNFD in one step", "POST", - "/vnfpkgm/v1/vnf_packages_content", headers_zip_yaml, "@b" + vnfd_filename_path, 201, - {"Location": "/vnfpkgm/v1/vnf_packages_content/", "Content-Type": "application/yaml"}, "yaml") + engine.test("Onboard empty VNFD in one step", "POST", "/vnfpkgm/v1/vnf_packages_content", headers_yaml, + self.vnfd_empty, 201, r_headers_yaml_location_vnfd, "yaml") self.vnfd_id = engine.last_id + # test bug 605 + engine.test("Upload invalid VNFD ", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id), + headers_yaml, self.vnfd_prova, 422, r_header_yaml, "yaml") + + engine.test("Upload VNFD {}".format(self.vnfd_filename), "PUT", + "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id), headers_zip_yaml, + "@b" + vnfd_filename_path, 204, None, 0) + + # test bug 605 + engine.test("Upload invalid VNFD ", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id), + headers_yaml, self.vnfd_prova, 422, r_header_yaml, "yaml") + # get vnfd descriptor - engine.test("Get VNFD descriptor", "GET", - "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id), headers_yaml, None, 200, r_header_yaml, "yaml") + engine.test("Get VNFD descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id), + headers_yaml, None, 200, r_header_yaml, "yaml") # get vnfd file descriptor - engine.test("Get VNFD file descriptor", "GET", - "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(self.vnfd_id), headers_text, None, 200, - r_header_text, "text", temp_dir+"vnfd-yaml") + engine.test("Get VNFD file descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(self.vnfd_id), + headers_text, None, 200, r_header_text, "text", temp_dir+"vnfd-yaml") # TODO compare files: diff vnfd-yaml hackfest_3charmed_vnfd/hackfest_3charmed_vnfd.yaml # get vnfd zip file package @@ -1682,25 +1707,22 @@ class TestDescriptors: # TODO compare files: diff vnfd-icon hackfest_3charmed_vnfd/icons/osm.png # nsd CREATE AND UPLOAD in one step: - engine.test("Onboard NSD in one step", "POST", - "/nsd/v1/ns_descriptors_content", headers_zip_yaml, "@b" + nsd_filename_path, 201, - {"Location": "/nsd/v1/ns_descriptors_content/", "Content-Type": "application/yaml"}, "yaml") + engine.test("Onboard NSD in one step", "POST", "/nsd/v1/ns_descriptors_content", headers_zip_yaml, + "@b" + nsd_filename_path, 201, r_headers_yaml_location_nsd, "yaml") self.nsd_id = engine.last_id # get nsd descriptor - engine.test("Get NSD descriptor", "GET", - "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml, None, 200, r_header_yaml, "yaml") + engine.test("Get NSD descriptor", "GET", "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml, + None, 200, r_header_yaml, "yaml") # get nsd file descriptor - engine.test("Get NSD file descriptor", "GET", - "/nsd/v1/ns_descriptors/{}/nsd".format(self.nsd_id), headers_text, None, 200, - r_header_text, "text", temp_dir+"nsd-yaml") + engine.test("Get NSD file descriptor", "GET", "/nsd/v1/ns_descriptors/{}/nsd".format(self.nsd_id), headers_text, + None, 200, r_header_text, "text", temp_dir+"nsd-yaml") # TODO compare files: diff nsd-yaml hackfest_3charmed_nsd/hackfest_3charmed_nsd.yaml # get nsd zip file package - engine.test("Get NSD zip package", "GET", - "/nsd/v1/ns_descriptors/{}/nsd_content".format(self.nsd_id), headers_zip, None, 200, - r_header_zip, "zip", temp_dir+"nsd-zip") + engine.test("Get NSD zip package", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(self.nsd_id), + headers_zip, None, 200, r_header_zip, "zip", temp_dir+"nsd-zip") # TODO compare files: diff nsd-zip hackfest_3charmed_nsd.tar.gz # get nsd artifact @@ -1710,15 +1732,15 @@ class TestDescriptors: # TODO compare files: diff nsd-icon hackfest_3charmed_nsd/icons/osm.png # vnfd DELETE - test_rest.test("Delete VNFD conflict", "DELETE", - "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id), headers_yaml, None, 409, None, None) + test_rest.test("Delete VNFD conflict", "DELETE", "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id), + headers_yaml, None, 409, None, None) - test_rest.test("Delete VNFD force", "DELETE", - "/vnfpkgm/v1/vnf_packages/{}?FORCE=TRUE".format(self.vnfd_id), headers_yaml, None, 204, None, 0) + test_rest.test("Delete VNFD force", "DELETE", "/vnfpkgm/v1/vnf_packages/{}?FORCE=TRUE".format(self.vnfd_id), + headers_yaml, None, 204, None, 0) # nsd DELETE - test_rest.test("Delete NSD", "DELETE", - "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml, None, 204, None, 0) + test_rest.test("Delete NSD", "DELETE", "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml, None, 204, + None, 0) class TestNetSliceTemplates: @@ -1732,7 +1754,7 @@ class TestNetSliceTemplates: engine.set_test_name("NST") engine.get_autorization() engine.test("Onboard NST", "POST", "/nst/v1/netslice_templates_content", headers_yaml, self.nst_filenames, - 201, {"Location": "/nst/v1/netslice_templates_content", "Content-Type": "application/yaml"}, "yaml") + 201, r_headers_yaml_location_nst, "yaml") nst_id = engine.last_id # nstd SHOW OSM format @@ -1756,7 +1778,7 @@ class TestNetSliceInstances: engine.set_test_name("NSI") engine.get_autorization() engine.test("Onboard NST", "POST", "/nst/v1/netslice_templates_content", headers_yaml, self.nst_filenames, 201, - {"Location": "/nst/v1/netslice_templates_content", "Content-Type": "application/yaml"}, "yaml") + r_headers_yaml_location_nst, "yaml") nst_id = engine.last_id # nsi CREATE @@ -1767,7 +1789,7 @@ class TestNetSliceInstances: ns_data_text = yaml.safe_dump(ns_data, default_flow_style=True, width=256) engine.test("Onboard NSI", "POST", "/nsilcm/v1/netslice_instances_content", headers_yaml, ns_data_text, 201, - {"Location": "/nsilcm/v1/netslice_instances_content", "Content-Type": "application/yaml"}, "yaml") + r_headers_yaml_location_nst, "yaml") nsi_id = engine.last_id # TODO: Improve the wait with a polling if NSI was deployed