From 21f3621e77e3aa027cefaa7da3b39795502b9219 Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Tue, 17 Jul 2018 15:17:09 -0500 Subject: [PATCH 1/5] Remove baseline checking logic The logic for rechecking baselines is no longer needed. Fixes #106. Signed-off-by: Major Hayden --- sktm/__init__.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index 492cfc0..7087691 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -369,22 +369,12 @@ def check_pending(self): logging.info("url=%s", rurl) basehash = self.jk.get_base_hash(self.jobname, bid) logging.info("basehash=%s", basehash) - if bres == sktm.tresult.BASELINE_FAILURE: - self.db.update_baseline( - self.baserepo, - basehash, - self.jk.get_base_commitdate(self.jobname, bid), - sktm.tresult.TEST_FAILURE, - bid - ) patch_url_list = self.jk.get_patchwork(self.jobname, bid) for patch_url in patch_url_list: patches.append(self.get_patch_info_from_url(cpw, patch_url)) - - if bres != sktm.tresult.BASELINE_FAILURE: - self.db.commit_tested(patches) + self.db.commit_tested(patches) else: raise Exception("Unknown job type: %d" % pjt) From e5583ce8ebba076606dbedf95e0015baefd27d99 Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Tue, 17 Jul 2018 15:20:33 -0500 Subject: [PATCH 2/5] Rename variables in check_pending() Rename variables so they are easier to read and understand. Signed-off-by: Major Hayden --- sktm/__init__.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index 7087691..4f67502 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -349,34 +349,38 @@ def check_patchwork(self): series.get_patch_url_list()) def check_pending(self): - for (pjt, bid, cpw) in self.pj: - if self.jk.is_build_complete(self.jobname, bid): - logging.info("job completed: jjid=%d; type=%d", bid, pjt) - self.pj.remove((pjt, bid, cpw)) - if pjt == sktm.jtype.BASELINE: + for (job_type, build_id, pw_instance) in self.pj: + if self.jk.is_build_complete(self.jobname, build_id): + logging.info("job completed: jjid=%d; type=%d", build_id, + job_type) + self.pj.remove((job_type, build_id, pw_instance)) + if job_type == sktm.jtype.BASELINE: self.db.update_baseline( self.baserepo, - self.jk.get_base_hash(self.jobname, bid), - self.jk.get_base_commitdate(self.jobname, bid), - self.jk.get_result(self.jobname, bid), - bid + self.jk.get_base_hash(self.jobname, build_id), + self.jk.get_base_commitdate(self.jobname, build_id), + self.jk.get_result(self.jobname, build_id), + build_id ) - elif pjt == sktm.jtype.PATCHWORK: + elif job_type == sktm.jtype.PATCHWORK: patches = list() - bres = self.jk.get_result(self.jobname, bid) - rurl = self.jk.get_result_url(self.jobname, bid) - logging.info("result=%s", bres) - logging.info("url=%s", rurl) - basehash = self.jk.get_base_hash(self.jobname, bid) + build_result = self.jk.get_result(self.jobname, build_id) + report_url = self.jk.get_result_url(self.jobname, build_id) + logging.info("result=%s", build_result) + logging.info("url=%s", report_url) + basehash = self.jk.get_base_hash(self.jobname, build_id) logging.info("basehash=%s", basehash) - patch_url_list = self.jk.get_patchwork(self.jobname, bid) + patch_url_list = self.jk.get_patchwork(self.jobname, + build_id) for patch_url in patch_url_list: - patches.append(self.get_patch_info_from_url(cpw, - patch_url)) + patches.append( + self.get_patch_info_from_url(pw_instance, + patch_url) + ) self.db.commit_tested(patches) else: - raise Exception("Unknown job type: %d" % pjt) + raise Exception("Unknown job type: %d" % job_type) def wait_for_pending(self): self.check_pending() From 1024c781f58f8b8215a1493a2a2c795c2e70e9a6 Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Tue, 17 Jul 2018 15:25:13 -0500 Subject: [PATCH 3/5] Wrap job completion check in try/except Ensure that a temporary network hiccup will not cause sktm to mark an entire test as failed. If a build completion check fails, retry the check during the next loop. Signed-off-by: Major Hayden --- sktm/__init__.py | 68 +++++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index 4f67502..1426ad5 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -350,37 +350,45 @@ def check_patchwork(self): def check_pending(self): for (job_type, build_id, pw_instance) in self.pj: - if self.jk.is_build_complete(self.jobname, build_id): - logging.info("job completed: jjid=%d; type=%d", build_id, - job_type) - self.pj.remove((job_type, build_id, pw_instance)) - if job_type == sktm.jtype.BASELINE: - self.db.update_baseline( - self.baserepo, - self.jk.get_base_hash(self.jobname, build_id), - self.jk.get_base_commitdate(self.jobname, build_id), - self.jk.get_result(self.jobname, build_id), - build_id + + try: + if not self.jk.is_build_complete(self.jobname, build_id): + continue + except: # noqa pylint: disable=bare-except + continue + + # Log the completed job and remove it from the list of jobs to + # check + logging.info("job completed: jjid=%d; type=%d", build_id, job_type) + self.pj.remove((job_type, build_id, pw_instance)) + + if job_type == sktm.jtype.BASELINE: + # Update baseline records + self.db.update_baseline( + self.baserepo, + self.jk.get_base_hash(self.jobname, build_id), + self.jk.get_base_commitdate(self.jobname, build_id), + self.jk.get_result(self.jobname, build_id), + build_id + ) + elif job_type == sktm.jtype.PATCHWORK: + # Update the testing status for the patchwork patches + patches = list() + build_result = self.jk.get_result(self.jobname, build_id) + report_url = self.jk.get_result_url(self.jobname, build_id) + logging.info("result=%s", build_result) + logging.info("url=%s", report_url) + basehash = self.jk.get_base_hash(self.jobname, build_id) + logging.info("basehash=%s", basehash) + + patch_url_list = self.jk.get_patchwork(self.jobname, build_id) + for patch_url in patch_url_list: + patches.append( + self.get_patch_info_from_url(pw_instance, patch_url) ) - elif job_type == sktm.jtype.PATCHWORK: - patches = list() - build_result = self.jk.get_result(self.jobname, build_id) - report_url = self.jk.get_result_url(self.jobname, build_id) - logging.info("result=%s", build_result) - logging.info("url=%s", report_url) - basehash = self.jk.get_base_hash(self.jobname, build_id) - logging.info("basehash=%s", basehash) - - patch_url_list = self.jk.get_patchwork(self.jobname, - build_id) - for patch_url in patch_url_list: - patches.append( - self.get_patch_info_from_url(pw_instance, - patch_url) - ) - self.db.commit_tested(patches) - else: - raise Exception("Unknown job type: %d" % job_type) + self.db.commit_tested(patches) + else: + raise Exception("Unknown job type: %d" % job_type) def wait_for_pending(self): self.check_pending() From 23da2b7d71634c24cfee23049f277d411e944b90 Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Tue, 17 Jul 2018 15:26:34 -0500 Subject: [PATCH 4/5] Add docstring for check_pending() Signed-off-by: Major Hayden --- sktm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sktm/__init__.py b/sktm/__init__.py index 1426ad5..4319415 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -349,6 +349,7 @@ def check_patchwork(self): series.get_patch_url_list()) def check_pending(self): + """Check pending jobs and update their status in the database.""" for (job_type, build_id, pw_instance) in self.pj: try: From ec1c9f606056488ae6259afbc3f8f96d73aa960f Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Tue, 17 Jul 2018 15:31:11 -0500 Subject: [PATCH 5/5] Refactor check_pending() Add comments, remove the if/elif/else, and create the `patches` variable using dictionary comprehension. Signed-off-by: Major Hayden --- sktm/__init__.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index 4319415..c626afc 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -363,8 +363,8 @@ def check_pending(self): logging.info("job completed: jjid=%d; type=%d", build_id, job_type) self.pj.remove((job_type, build_id, pw_instance)) + # Update baseline records if job_type == sktm.jtype.BASELINE: - # Update baseline records self.db.update_baseline( self.baserepo, self.jk.get_base_hash(self.jobname, build_id), @@ -372,24 +372,36 @@ def check_pending(self): self.jk.get_result(self.jobname, build_id), build_id ) - elif job_type == sktm.jtype.PATCHWORK: - # Update the testing status for the patchwork patches - patches = list() + continue + + # Update the testing status for the patchwork patches + if job_type == sktm.jtype.PATCHWORK: + # Retrieve the result of the Jenkins build build_result = self.jk.get_result(self.jobname, build_id) - report_url = self.jk.get_result_url(self.jobname, build_id) logging.info("result=%s", build_result) + + # Get the URL of the Jenkins build + report_url = self.jk.get_result_url(self.jobname, build_id) logging.info("url=%s", report_url) + + # Note the base hash of the kernel repository that the patches + # were applied to basehash = self.jk.get_base_hash(self.jobname, build_id) logging.info("basehash=%s", basehash) + # Get a list of the patchwork URLs in the job and commit them + # to the `patch` table within the database patch_url_list = self.jk.get_patchwork(self.jobname, build_id) - for patch_url in patch_url_list: - patches.append( - self.get_patch_info_from_url(pw_instance, patch_url) - ) + patches = [ + self.get_patch_info_from_url(pw_instance, x) + for x in patch_url_list + ] self.db.commit_tested(patches) - else: - raise Exception("Unknown job type: %d" % job_type) + + continue + + # sktm does not know how to handle this job type + raise Exception("Unknown job type: %d" % job_type) def wait_for_pending(self): self.check_pending()