From 99a9e9cd8dfce78c825552872d0b1cfa9cea5278 Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Mon, 26 Sep 2022 07:25:40 -0700 Subject: [PATCH 1/4] [ZTP] Improvements to allow in-band zero touch provisioning - Download all plugins before processing each configuration sections. This ensures that configuration section plugins are available at all times. If the plugin of a configuration section needs to be downloaded at a later time when ZTP is in-progress and the configuration section is actually processed, set the field "pre-ztp-plugin-download" : false in the corresponding configuration section data in ztp.json. - Additional unit test cases added --- .../python3/dist-packages/ztp/Downloader.py | 6 +- .../python3/dist-packages/ztp/ZTPSections.py | 2 +- .../lib/python3/dist-packages/ztp/defaults.py | 1 + src/usr/lib/ztp/ztp-engine.py | 92 +++++++++++++++---- tests/test_ZTPJson.py | 18 ++++ tests/test_ztp_engine.py | 41 +++++++++ 6 files changed, 139 insertions(+), 21 deletions(-) diff --git a/src/usr/lib/python3/dist-packages/ztp/Downloader.py b/src/usr/lib/python3/dist-packages/ztp/Downloader.py index 7f3348d..08ef3d1 100644 --- a/src/usr/lib/python3/dist-packages/ztp/Downloader.py +++ b/src/usr/lib/python3/dist-packages/ztp/Downloader.py @@ -247,6 +247,10 @@ def getUrl(self, url=None, dst_file=None, incl_http_headers=None, is_secure=True else: break - os.chmod(dst_file, stat.S_IRWXU) + try: + os.chmod(dst_file, stat.S_IRWXU) + except FileNotFoundError: + return (20, None) + # Use curl result return (0, dst_file) diff --git a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py index 0da76d3..a5fee48 100644 --- a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py +++ b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py @@ -91,7 +91,7 @@ def __buildDefaults(self, section): @param section (dict) Configuration Section input data read from JSON file. ''' - default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure'] + default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure', 'pre-ztp-plugin-download'] # Loop through objects and update them with default values for key in default_objs: _val = getField(section, key, bool, getCfg(key)) diff --git a/src/usr/lib/python3/dist-packages/ztp/defaults.py b/src/usr/lib/python3/dist-packages/ztp/defaults.py index 34464d3..710375e 100644 --- a/src/usr/lib/python3/dist-packages/ztp/defaults.py +++ b/src/usr/lib/python3/dist-packages/ztp/defaults.py @@ -48,6 +48,7 @@ "log-file" : "/var/log/ztp.log", \ "log-level" : "INFO", \ "monitor-startup-config" : True, \ + "pre-ztp-plugin-download" : True, \ "restart-ztp-interval": 300, \ "reboot-on-success" : False, \ "reboot-on-failure" : False, \ diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index 1d6ad89..4bdfa9c 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -432,13 +432,55 @@ def __evalZTPResult(self): # Check reboot on result flags and take action self.__rebootAction(self.objztpJson.ztpDict, delayed_reboot=True) + def __downloadPlugins(self): + '''! + Check and download plugins used by configuration sections + @return False - If failed to download one more plugins of a required configuration section + True - Successfully downloaded plugins of all configuration sections + ''' + + # Obtain a copy of the list of configuration sections + section_names = list(self.objztpJson.section_names) + abort = False + logger.debug('Verifying and downloading plugins user by configuration sections: %s' % ', '.join(section_names)) + for sec in sorted(section_names): + section = self.objztpJson.ztpDict.get(sec) + t = getTimestamp() + try: + # Retrieve individual section's progress + sec_status = section.get('status') + download_check = section.get('pre-ztp-plugin-download') + if sec_status == 'BOOT' and download_check: + logger.info('Verifying and downloading plugin used by the configuration section %s.' % (sec)) + updateActivity('Verifying and downloading plugin used by the configuration section %s' % sec) + # Get the appropriate plugin to be used for this configuration section + plugin = self.objztpJson.plugin(sec) + if plugin is None: + # Mark section status as failed + section['error'] = 'Unable to find or download requested plugin' + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + + except: + logger.debug('Exception [%s] encountered while verifying plugin for configuration section %s.' % (str(e), sec)) + logger.info('Exception encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % sec) + section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + return abort + def __processConfigSections(self): '''! Process and execute individual configuration sections defined in ZTP JSON. Plugin for each configuration section is resolved and executed. Configuration section data is provided as command line argument to the plugin. Each and every section is processed before this function returns. - + @return False - If error encountered processing configuration sections and request restarting ZTP + True - If processing of configuration sections has been completed ''' # Obtain a copy of the list of configuration sections @@ -447,6 +489,9 @@ def __processConfigSections(self): # set temporary flags abort = False sort = True + if self.__downloadPlugins(): + logger.info('Halting ZTP as download of one or more plugins FAILED.') + return False logger.debug('Processing configuration sections: %s' % ', '.join(section_names)) # Loop through each sections till all of them are processed @@ -537,6 +582,7 @@ def __processConfigSections(self): # Check reboot on result flags self.__rebootAction(section) + return True def __processZTPJson(self): '''! @@ -598,32 +644,40 @@ def __processZTPJson(self): self.__loadZTPProfile("resume") # Process available configuration sections in ZTP JSON - self.__processConfigSections() - - # Determine ZTP result - self.__evalZTPResult() - - # Check restart ZTP condition - # ZTP result is failed and restart-ztp-on-failure is set or - _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ - self.objztpJson['restart-ztp-on-failure'] == True) - - # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set - _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ - self.objztpJson['restart-ztp-no-config'] == True and \ - self.objztpJson['config-fallback'] == False and - os.path.isfile(getCfg('config-db-json')) is False ) + _processing_completed = self.__processConfigSections() + # In test mode always mark processing as completed + if self.test_mode: + _processing_completed = True + + _restart_ztp_missing_config = False + _restart_ztp_on_failure = False + if _processing_completed: + # Determine ZTP result + self.__evalZTPResult() + + # Check restart ZTP condition + # ZTP result is failed and restart-ztp-on-failure is set or + _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ + self.objztpJson['restart-ztp-on-failure'] == True) + + # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set + _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ + self.objztpJson['restart-ztp-no-config'] == True and \ + self.objztpJson['config-fallback'] == False and + os.path.isfile(getCfg('config-db-json')) is False ) # Mark ZTP for restart - if _restart_ztp_missing_config or _restart_ztp_on_failure: + if not _processing_completed or _restart_ztp_missing_config or _restart_ztp_on_failure: os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): - os.remove(getCfg('ztp-json-shadow')) + os.remove(getCfg('ztp-json-shadow')) self.objztpJson = None # Remove startup-config file to obtain a new one through ZTP if getCfg('monitor-startup-config') is True and os.path.isfile(getCfg('config-db-json')): os.remove(getCfg('config-db-json')) - if _restart_ztp_missing_config: + if not _processing_completed: + return ("restart", "Restarting ZTP due to error processing configuration sections") + elif _restart_ztp_missing_config: return ("restart", "ZTP completed but startup configuration '%s' not found" % (getCfg('config-db-json'))) elif _restart_ztp_on_failure: return ("restart", "ZTP completed with FAILED status") diff --git a/tests/test_ZTPJson.py b/tests/test_ZTPJson.py index b008de8..c38e215 100644 --- a/tests/test_ZTPJson.py +++ b/tests/test_ZTPJson.py @@ -191,6 +191,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "source": "/tmp/test_firmware_%s.json" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -199,6 +200,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -305,6 +307,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -313,6 +316,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -459,6 +463,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "source": "file:///tmp/test_firmware.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -467,6 +472,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -526,6 +532,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): "destination": "/tmp/firmware_check.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -533,6 +540,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -682,6 +690,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -689,6 +698,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -765,6 +775,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -772,6 +783,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -813,6 +825,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -820,6 +833,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -874,6 +888,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -881,6 +896,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -927,6 +943,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "halt-on-failure": false, "ignore-result": false, "plugin": "graphservice", + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -935,6 +952,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, diff --git a/tests/test_ztp_engine.py b/tests/test_ztp_engine.py index 7d73d9c..091c028 100644 --- a/tests/test_ztp_engine.py +++ b/tests/test_ztp_engine.py @@ -448,6 +448,47 @@ def test_ztp_json_ignore_fail(self): self.cfgSet('monitor-startup-config', True) self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_invalid_plugins(self): + '''! + Simple ZTP test with 3-sections, invalid plugin in the second section, ZTP Failed + ''' + content = """{ + "ztp": { + "0001-test-plugin": { + "message" : "0001-test-plugin", + "message-file" : "/etc/ztp.results" + }, + "0002-invalid-plugin": { + "pre-ztp-plugin-download" : false + }, + "0003-test-plugin": { + "pre-ztp-plugin-download" : false, + "message" : "0003-test-plugin", + "message-file" : "/etc/ztp.results" + } + } +}""" + expected_result = """0001-test-plugin +0003-test-plugin +""" + self.__init_ztp_data() + self.cfgSet('monitor-startup-config', False) + self.cfgSet('restart-ztp-no-config', False) + self.__write_file("/tmp/ztp_input.json", content) + self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json") + runCommand(COVERAGE + ZTP_ENGINE_CMD) + runCommand(COVERAGE + ZTP_CMD + ' status -v') + os.remove("/tmp/ztp_input.json") + objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4) + assert(jsonDict.get('ztp').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin') + result = self.__read_file("/etc/ztp.results") + assert(result == expected_result) + self.cfgSet('monitor-startup-config', True) + self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_ignore_ztp_success(self): '''! Simple ZTP test with 3-sections, Failure in 3 sections but ignore-result set in 2sections, ignore-result set in ztp, ZTP Success From 36327a702c1295b9b75c2a0f8c113a1f9d44946b Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Mon, 26 Sep 2022 08:04:19 -0700 Subject: [PATCH 2/4] [ZTP] Fix python lgtm alerts --- src/usr/lib/ztp/ztp-engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index 4bdfa9c..c2cc6ab 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -463,7 +463,7 @@ def __downloadPlugins(self): if not section.get('ignore-result'): abort = True - except: + except Exception as e: logger.debug('Exception [%s] encountered while verifying plugin for configuration section %s.' % (str(e), sec)) logger.info('Exception encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % sec) section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) From e7288706afea614e7c0cfe1779d63368f5109c8b Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Fri, 30 Sep 2022 11:25:28 -0700 Subject: [PATCH 3/4] Review comments --- src/usr/lib/ztp/ztp-engine.py | 3 +-- tests/test_ztp_engine.py | 46 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index c2cc6ab..bc272fd 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -464,8 +464,7 @@ def __downloadPlugins(self): abort = True except Exception as e: - logger.debug('Exception [%s] encountered while verifying plugin for configuration section %s.' % (str(e), sec)) - logger.info('Exception encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % sec) + logger.error('Exception [%s] encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % (str(e), sec)) section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) section['start-timestamp'] = t self.objztpJson.updateStatus(section, 'FAILED') diff --git a/tests/test_ztp_engine.py b/tests/test_ztp_engine.py index 091c028..99a32fd 100644 --- a/tests/test_ztp_engine.py +++ b/tests/test_ztp_engine.py @@ -489,6 +489,52 @@ def test_ztp_json_invalid_plugins(self): self.cfgSet('monitor-startup-config', True) self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_invalid_plugins_ignore_result(self): + '''! + Simple ZTP test with 3-sections, invalid plugin in the second section but result ignored, + ZTP Success + ''' + content = """{ + "ztp": { + "0001-test-plugin": { + "message" : "0001-test-plugin", + "message-file" : "/etc/ztp.results" + }, + "0002-invalid-plugin": { + "plugin" : { + "url" : "file:///no-such-plugin" + }, + "pre-ztp-plugin-download" : false, + "ignore-result" : true + }, + "0003-test-plugin": { + "pre-ztp-plugin-download" : false, + "message" : "0003-test-plugin", + "message-file" : "/etc/ztp.results" + } + } +}""" + expected_result = """0001-test-plugin +0003-test-plugin +""" + self.__init_ztp_data() + self.cfgSet('monitor-startup-config', False) + self.cfgSet('restart-ztp-no-config', False) + self.__write_file("/tmp/ztp_input.json", content) + self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json") + runCommand(COVERAGE + ZTP_ENGINE_CMD) + runCommand(COVERAGE + ZTP_CMD + ' status -v') + os.remove("/tmp/ztp_input.json") + objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4) + assert(jsonDict.get('ztp').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin') + result = self.__read_file("/etc/ztp.results") + assert(result == expected_result) + self.cfgSet('monitor-startup-config', True) + self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_ignore_ztp_success(self): '''! Simple ZTP test with 3-sections, Failure in 3 sections but ignore-result set in 2sections, ignore-result set in ztp, ZTP Success From 01d1777a711b9aff4b2a3ca319e02b26b6183d8b Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Thu, 6 Oct 2022 11:56:23 -0700 Subject: [PATCH 4/4] Fix typo in plugin download error log message --- src/usr/lib/ztp/ztp-engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index bc272fd..7c229d9 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -464,7 +464,7 @@ def __downloadPlugins(self): abort = True except Exception as e: - logger.error('Exception [%s] encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % (str(e), sec)) + logger.error('Exception [%s] encountered while downloading plugin for configuration section %s. Marking it as FAILED.' % (str(e), sec)) section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) section['start-timestamp'] = t self.objztpJson.updateStatus(section, 'FAILED')