From b1ca0a40d4b9ec8d3e5bb39b8aaec89ef8d644d0 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 13:05:34 +0300 Subject: [PATCH 1/8] Move tresult into sktm.misc Move sktm.tresult to sktm.misc.tresult to fix composition, so that modules lower in the hierarchy (e.g. sktm.jenkins) do not have to import the module above (i.e. sktm). Fixes #6 --- sktm/__init__.py | 17 ++++------------- sktm/db.py | 6 +++--- sktm/jenkins.py | 16 ++++++++-------- sktm/misc.py | 25 +++++++++++++++++++++++++ sktm/patchwork.py | 14 +++++++------- 5 files changed, 47 insertions(+), 31 deletions(-) create mode 100644 sktm/misc.py diff --git a/sktm/__init__.py b/sktm/__init__.py index 30b34cc..d4237ee 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -20,19 +20,10 @@ import time import sktm.db import sktm.jenkins +import sktm.misc import sktm.patchwork -class tresult(enum.IntEnum): - """Test result""" - SUCCESS = 0 - MERGE_FAILURE = 1 - BUILD_FAILURE = 2 - PUBLISH_FAILURE = 3 - TEST_FAILURE = 4 - BASELINE_FAILURE = 5 - - class jtype(enum.IntEnum): """Job type""" BASELINE = 0 @@ -303,12 +294,12 @@ def check_pending(self): logging.info("url=%s", rurl) basehash = self.jk.get_base_hash(self.jobname, bid) logging.info("basehash=%s", basehash) - if bres == sktm.tresult.BASELINE_FAILURE: + if bres == sktm.misc.tresult.BASELINE_FAILURE: self.db.update_baseline( self.baserepo, basehash, self.jk.get_base_commitdate(self.jobname, bid), - sktm.tresult.TEST_FAILURE, + sktm.misc.tresult.TEST_FAILURE, bid ) @@ -342,7 +333,7 @@ def check_pending(self): except ValueError: pass - if bres != sktm.tresult.BASELINE_FAILURE: + if bres != sktm.misc.tresult.BASELINE_FAILURE: self.db.commit_patchtest(self.baserepo, basehash, patches, bres, bid, series) else: diff --git a/sktm/db.py b/sktm/db.py index 2b6d4a6..4447c5b 100644 --- a/sktm/db.py +++ b/sktm/db.py @@ -17,7 +17,7 @@ import os import sqlite3 import time -import sktm +import sktm.misc class skt_db(object): @@ -350,7 +350,7 @@ def get_baselineresult(self, baserepo, commithash): if not result: return None - return sktm.tresult(result[0]) + return sktm.misc.tresult(result[0]) def get_stable(self, baserepo): """Get the latest stable commit ID for a baseline Git repo URL. @@ -631,7 +631,7 @@ def dump_baseline_tests(self): # pragma: no cover for (burl, commit, res, buildid) in self.cur.fetchall(): print("repo url:", burl) print("commit id:", commit) - print("result:", sktm.tresult(res).name) + print("result:", sktm.misc.tresult(res).name) print("build id: #", buildid, sep='') print("---") diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 11951fe..1fed91e 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -18,7 +18,7 @@ import jenkinsapi -import sktm +import sktm.misc class skt_jenkins(object): @@ -138,7 +138,7 @@ def get_result(self, jobname, buildid): logging.info("build_status=%s", bstatus) if bstatus == "SUCCESS": - return sktm.tresult.SUCCESS + return sktm.misc.tresult.SUCCESS if not build.has_resultset(): raise Exception("No results for build %d (%s)" % @@ -149,9 +149,9 @@ def get_result(self, jobname, buildid): ["PASSED", "FIXED"]): if self.get_baseretcode(jobname, buildid) != 0: logging.warning("baseline failure found during patch testing") - return sktm.tresult.BASELINE_FAILURE + return sktm.misc.tresult.BASELINE_FAILURE - return sktm.tresult.SUCCESS + return sktm.misc.tresult.SUCCESS for (key, val) in build.get_resultset().iteritems(): if not key.startswith("skt."): @@ -160,14 +160,14 @@ def get_result(self, jobname, buildid): logging.debug("key=%s; value=%s", key, val.status) if val.status == "FAILED" or val.status == "REGRESSION": if key == "skt.cmd_merge": - return sktm.tresult.MERGE_FAILURE + return sktm.misc.tresult.MERGE_FAILURE elif key == "skt.cmd_build": - return sktm.tresult.BUILD_FAILURE + return sktm.misc.tresult.BUILD_FAILURE elif key == "skt.cmd_run": - return sktm.tresult.TEST_FAILURE + return sktm.misc.tresult.TEST_FAILURE logging.warning("Unknown status. marking as test failure") - return sktm.tresult.TEST_FAILURE + return sktm.misc.tresult.TEST_FAILURE # FIXME Clarify/fix argument names def build(self, jobname, baserepo=None, ref=None, baseconfig=None, diff --git a/sktm/misc.py b/sktm/misc.py new file mode 100644 index 0000000..c4fc7fd --- /dev/null +++ b/sktm/misc.py @@ -0,0 +1,25 @@ +# Copyright (c) 2017-2018 Red Hat, Inc. All rights reserved. This copyrighted +# material is made available to anyone wishing to use, modify, copy, or +# redistribute it subject to the terms and conditions of the GNU General +# Public License v.2 or later. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import enum + + +class tresult(enum.IntEnum): + """Test result""" + SUCCESS = 0 + MERGE_FAILURE = 1 + BUILD_FAILURE = 2 + PUBLISH_FAILURE = 3 + TEST_FAILURE = 4 + BASELINE_FAILURE = 5 diff --git a/sktm/patchwork.py b/sktm/patchwork.py index 1ef9a75..718d23a 100644 --- a/sktm/patchwork.py +++ b/sktm/patchwork.py @@ -24,7 +24,7 @@ import re import urllib import xmlrpclib -import sktm +import sktm.misc class ObjectSummary(object): @@ -588,12 +588,12 @@ def _set_patch_check(self, patch, payload): def set_patch_check(self, pid, jurl, result): """ Add a patch "check" for the specified patch, with the specified - Jenkins build URL and result (sktm.tresult). + Jenkins build URL and result (sktm.misc.tresult). Args: pid: The ID of the patch to add the "check" for. jurl: Jenkins build URL for the "check" to reference. - result: Test result (sktm.tresult) to feature in the "check" + result: Test result (sktm.misc.tresult) to feature in the "check" state. """ if self.apikey is None: @@ -605,9 +605,9 @@ def set_patch_check(self, pid, jurl, result): 'target_url': jurl, 'context': 'skt', 'description': 'skt boot test'} - if result == sktm.tresult.SUCCESS: + if result == sktm.misc.tresult.SUCCESS: payload['state'] = int(pwresult.SUCCESS) - elif result == sktm.tresult.BASELINE_FAILURE: + elif result == sktm.misc.tresult.BASELINE_FAILURE: payload['state'] = int(pwresult.WARNING) payload['description'] = 'Baseline failure found while testing ' 'this patch' @@ -1000,12 +1000,12 @@ def get_emails(self, pid): def set_patch_check(self, pid, jurl, result): """ Add a patch "check" for the specified patch, with the specified - Jenkins build URL and result (sktm.tresult). + Jenkins build URL and result (sktm.misc.tresult). Args: pid: The ID of the patch to add the "check" for. jurl: Jenkins build URL for the "check" to reference. - result: Test result (sktm.tresult) to feature in the "check" + result: Test result (sktm.misc.tresult) to feature in the "check" state. """ # TODO: Implement this for xmlrpc From 13d307ff76460229c0a95cc36a36b74c609c70a3 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 13:33:38 +0300 Subject: [PATCH 2/8] Describe sktm.jenkins.get_result* functions --- sktm/jenkins.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 1fed91e..74e1649 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -128,10 +128,33 @@ def get_baseretcode(self, jobname, buildid): "baseretcode", 0) def get_result_url(self, jobname, buildid): + """ + Get the URL of the web representation of the specified build of the + specified Jenkins project. + + Args: + jobname: Jenkins project name. + buildid: Jenkins build ID. + + Result: + The URL of the build result. + """ return "%s/job/%s/%s" % (self.server.base_server_url(), jobname, buildid) def get_result(self, jobname, buildid): + """ + Get result code (sktm.misc.tresult) for the specified build of the + specified Jenkins project. Wait for the build to complete, if it + hasn't yet. + + Args: + jobname: Jenkins project name. + buildid: Jenkins build ID. + + Result: + The build result code (sktm.misc.tresult). + """ build = self._wait_and_get_build(jobname, buildid) bstatus = build.get_status() From 29a6db246b3d97935c90ab6c86f3b2fafd7b5284 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 13:48:16 +0300 Subject: [PATCH 3/8] Convert Jenkins interface to Jenkins project interface Instead of creating and using a "Jenkins Interface" (sktm.jenkins.skt_jenkins), and then supplying a project name ("jobname") with every method call, create and use a "Jenkins Project Interface" (sktm.jenkins.Project) and only supply the project name on its creation. This simplifes the interface, and removes a bit of code. This also prepares for abstracting Jenkins away. --- sktm/__init__.py | 31 +++++++-------- sktm/jenkins.py | 100 +++++++++++++++++++++-------------------------- 2 files changed, 58 insertions(+), 73 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index d4237ee..a4aa4cd 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -58,11 +58,10 @@ def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, # FIXME Clarify/fix member variable names # Database instance self.db = sktm.db.skt_db(os.path.expanduser(dbpath)) - # Jenkins interface instance - self.jk = sktm.jenkins.skt_jenkins(jenkinsurl, jenkinslogin, - jenkinspassword) - # Jenkins project name - self.jobname = jenkinsjobname + # Jenkins project interface instance + self.jk = sktm.jenkins.JenkinsProject(jenkinsjobname, + jenkinsurl, jenkinslogin, + jenkinspassword) # Patchset filter program self.filter = filter # Extra arguments to pass to "make" @@ -159,8 +158,7 @@ def add_pw(self, baseurl, pname, lpatch=None, apikey=None): def check_baseline(self): """Submit a build for baseline""" self.pj.append((sktm.jtype.BASELINE, - self.jk.build(self.jobname, - baserepo=self.baserepo, + self.jk.build(baserepo=self.baserepo, ref=self.baseref, baseconfig=self.cfgurl, makeopts=self.makeopts), @@ -255,7 +253,6 @@ def check_patchwork(self): # Submit and remember a Jenkins build for the patchset self.pj.append((sktm.jtype.PATCHWORK, self.jk.build( - self.jobname, baserepo=self.baserepo, ref=stablecommit, baseconfig=self.cfgurl, @@ -273,37 +270,37 @@ def check_patchwork(self): def check_pending(self): for (pjt, bid, cpw) in self.pj: - if self.jk.is_build_complete(self.jobname, bid): + if self.jk.is_build_complete(bid): logging.info("job completed: jjid=%d; type=%d", bid, pjt) self.pj.remove((pjt, bid, cpw)) if pjt == sktm.jtype.BASELINE: self.db.update_baseline( self.baserepo, - self.jk.get_base_hash(self.jobname, bid), - self.jk.get_base_commitdate(self.jobname, bid), - self.jk.get_result(self.jobname, bid), + self.jk.get_base_hash(bid), + self.jk.get_base_commitdate(bid), + self.jk.get_result(bid), bid ) elif pjt == sktm.jtype.PATCHWORK: patches = list() slist = list() series = None - bres = self.jk.get_result(self.jobname, bid) - rurl = self.jk.get_result_url(self.jobname, bid) + bres = self.jk.get_result(bid) + rurl = self.jk.get_result_url(bid) logging.info("result=%s", bres) logging.info("url=%s", rurl) - basehash = self.jk.get_base_hash(self.jobname, bid) + basehash = self.jk.get_base_hash(bid) logging.info("basehash=%s", basehash) if bres == sktm.misc.tresult.BASELINE_FAILURE: self.db.update_baseline( self.baserepo, basehash, - self.jk.get_base_commitdate(self.jobname, bid), + self.jk.get_base_commitdate(bid), sktm.misc.tresult.TEST_FAILURE, bid ) - patchset = self.jk.get_patchwork(self.jobname, bid) + patchset = self.jk.get_patchwork(bid) for purl in patchset: match = re.match(r"(.*)/patch/(\d+)$", purl) if match: diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 74e1649..5eb9430 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -21,22 +21,25 @@ import sktm.misc -class skt_jenkins(object): - """Jenkins interface""" - def __init__(self, url, username=None, password=None): +class JenkinsProject(object): + """Jenkins project interface""" + def __init__(self, name, url, username=None, password=None): """ - Initialize a Jenkins interface. + Initialize a Jenkins project interface. Args: + name: Name of the Jenkins project to operate on. url: Jenkins instance URL. username: Jenkins user name. password: Jenkins user password. """ + self.name = name + # Initialize Jenkins server interface # TODO Add support for CSRF protection self.server = jenkinsapi.jenkins.Jenkins(url, username, password) - def _wait_and_get_build(self, jobname, buildid): - job = self.server.get_job(jobname) + def _wait_and_get_build(self, buildid): + job = self.server.get_job(self.name) build = job.get_build(buildid) build.block_until_complete(delay=60) @@ -45,14 +48,13 @@ def _wait_and_get_build(self, jobname, buildid): return build - def get_cfg_data(self, jobname, buildid, stepname, cfgkey, default=None): + def get_cfg_data(self, buildid, stepname, cfgkey, default=None): """ Get a value from a JSON-formatted output of a test result, of the - specified completed build for the specified project. Wait for the - build to complete, if it hasn't yet. + specified completed build. Wait for the build to complete, if it + hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. stepname: Test (step) path in the result, which output should be parsed as JSON. @@ -63,7 +65,7 @@ def get_cfg_data(self, jobname, buildid, stepname, cfgkey, default=None): Returns: The key value, or the default if not found. """ - build = self._wait_and_get_build(jobname, buildid) + build = self._wait_and_get_build(buildid) if not build.has_resultset(): raise Exception("No results for build %d (%s)" % @@ -75,87 +77,74 @@ def get_cfg_data(self, jobname, buildid, stepname, cfgkey, default=None): cfg = json.loads(val.stdout) return cfg.get(cfgkey, default) - def get_base_commitdate(self, jobname, buildid): + def get_base_commitdate(self, buildid): """ - Get base commit's committer date of the specified completed build for - the specified project. Wait for the build to complete, if it hasn't - yet. + Get base commit's committer date of the specified completed build. + Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The epoch timestamp string of the committer date. """ - return self.get_cfg_data(jobname, buildid, "skt.cmd_merge", - "commitdate") + return self.get_cfg_data(buildid, "skt.cmd_merge", "commitdate") - def get_base_hash(self, jobname, buildid): + def get_base_hash(self, buildid): """ - Get base commit's hash of the specified completed build for the - specified project. Wait for the build to complete, if it hasn't yet. + Get base commit's hash of the specified completed build. + Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The base commit's hash string. """ - return self.get_cfg_data(jobname, buildid, "skt.cmd_merge", - "basehead") + return self.get_cfg_data(buildid, "skt.cmd_merge", "basehead") # FIXME Clarify function name - def get_patchwork(self, jobname, buildid): + def get_patchwork(self, buildid): """ - Get the list of Patchwork patch URLs for the specified completed build - for the specified project. Wait for the build to complete, if it - hasn't yet. + Get the list of Patchwork patch URLs for the specified completed + build. Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The list of Patchwork patch URLs. """ - return self.get_cfg_data(jobname, buildid, "skt.cmd_merge", - "pw") + return self.get_cfg_data(buildid, "skt.cmd_merge", "pw") - def get_baseretcode(self, jobname, buildid): - return self.get_cfg_data(jobname, buildid, "skt.cmd_run", - "baseretcode", 0) + def get_baseretcode(self, buildid): + return self.get_cfg_data(buildid, "skt.cmd_run", "baseretcode", 0) - def get_result_url(self, jobname, buildid): + def get_result_url(self, buildid): """ - Get the URL of the web representation of the specified build of the - specified Jenkins project. + Get the URL of the web representation of the specified build. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Result: The URL of the build result. """ - return "%s/job/%s/%s" % (self.server.base_server_url(), jobname, + return "%s/job/%s/%s" % (self.server.base_server_url(), self.name, buildid) - def get_result(self, jobname, buildid): + def get_result(self, buildid): """ - Get result code (sktm.misc.tresult) for the specified build of the - specified Jenkins project. Wait for the build to complete, if it - hasn't yet. + Get result code (sktm.misc.tresult) for the specified build. + Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Result: The build result code (sktm.misc.tresult). """ - build = self._wait_and_get_build(jobname, buildid) + build = self._wait_and_get_build(buildid) bstatus = build.get_status() logging.info("build_status=%s", bstatus) @@ -170,7 +159,7 @@ def get_result(self, jobname, buildid): if bstatus == "UNSTABLE" and \ (build.get_resultset()["skt.cmd_run"].status in ["PASSED", "FIXED"]): - if self.get_baseretcode(jobname, buildid) != 0: + if self.get_baseretcode(buildid) != 0: logging.warning("baseline failure found during patch testing") return sktm.misc.tresult.BASELINE_FAILURE @@ -193,14 +182,13 @@ def get_result(self, jobname, buildid): return sktm.misc.tresult.TEST_FAILURE # FIXME Clarify/fix argument names - def build(self, jobname, baserepo=None, ref=None, baseconfig=None, + def build(self, baserepo=None, ref=None, baseconfig=None, message_id=None, subject=None, emails=set(), patchwork=[], makeopts=None): """ Submit a build of a patchset. Args: - jobname: Name of the Jenkins project to build. baserepo: Baseline Git repo URL. ref: Baseline Git reference to test. baseconfig: Kernel configuration URL. @@ -243,15 +231,15 @@ def build(self, jobname, baserepo=None, ref=None, baseconfig=None, params["makeopts"] = makeopts logging.debug(params) - self.server.get_job(jobname) - expected_id = self.server.get_job(jobname).get_next_build_number() - self.server.build_job(jobname, params) - build = self.find_build(jobname, params, expected_id) + self.server.get_job(self.name) + expected_id = self.server.get_job(self.name).get_next_build_number() + self.server.build_job(self.name, params) + build = self.find_build(params, expected_id) logging.info("submitted build: %s", build) return build.get_number() - def is_build_complete(self, jobname, buildid): - job = self.server.get_job(jobname) + def is_build_complete(self, buildid): + job = self.server.get_job(self.name) build = job.get_build(buildid) return not build.is_running() @@ -269,8 +257,8 @@ def _params_eq(self, build, params): return True - def find_build(self, jobname, params, eid=None): - job = self.server.get_job(jobname) + def find_build(self, params, eid=None): + job = self.server.get_job(self.name) lbuild = None while not lbuild: From bbc4d6f83e258413a58cbada99fc50c141a72d1b Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 14:04:47 +0300 Subject: [PATCH 4/8] Mark sktm.jenkins.Project private methods Mark sktm.jenkins.Project private methods by prepending them with double underscore. Concerns #12 --- sktm/jenkins.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 5eb9430..7ecdf47 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -38,7 +38,7 @@ def __init__(self, name, url, username=None, password=None): # TODO Add support for CSRF protection self.server = jenkinsapi.jenkins.Jenkins(url, username, password) - def _wait_and_get_build(self, buildid): + def __wait_and_get_build(self, buildid): job = self.server.get_job(self.name) build = job.get_build(buildid) build.block_until_complete(delay=60) @@ -48,7 +48,7 @@ def _wait_and_get_build(self, buildid): return build - def get_cfg_data(self, buildid, stepname, cfgkey, default=None): + def __get_cfg_data(self, buildid, stepname, cfgkey, default=None): """ Get a value from a JSON-formatted output of a test result, of the specified completed build. Wait for the build to complete, if it @@ -65,7 +65,7 @@ def get_cfg_data(self, buildid, stepname, cfgkey, default=None): Returns: The key value, or the default if not found. """ - build = self._wait_and_get_build(buildid) + build = self.__wait_and_get_build(buildid) if not build.has_resultset(): raise Exception("No results for build %d (%s)" % @@ -88,7 +88,7 @@ def get_base_commitdate(self, buildid): Return: The epoch timestamp string of the committer date. """ - return self.get_cfg_data(buildid, "skt.cmd_merge", "commitdate") + return self.__get_cfg_data(buildid, "skt.cmd_merge", "commitdate") def get_base_hash(self, buildid): """ @@ -101,7 +101,7 @@ def get_base_hash(self, buildid): Return: The base commit's hash string. """ - return self.get_cfg_data(buildid, "skt.cmd_merge", "basehead") + return self.__get_cfg_data(buildid, "skt.cmd_merge", "basehead") # FIXME Clarify function name def get_patchwork(self, buildid): @@ -115,10 +115,10 @@ def get_patchwork(self, buildid): Return: The list of Patchwork patch URLs. """ - return self.get_cfg_data(buildid, "skt.cmd_merge", "pw") + return self.__get_cfg_data(buildid, "skt.cmd_merge", "pw") - def get_baseretcode(self, buildid): - return self.get_cfg_data(buildid, "skt.cmd_run", "baseretcode", 0) + def __get_baseretcode(self, buildid): + return self.__get_cfg_data(buildid, "skt.cmd_run", "baseretcode", 0) def get_result_url(self, buildid): """ @@ -144,7 +144,7 @@ def get_result(self, buildid): Result: The build result code (sktm.misc.tresult). """ - build = self._wait_and_get_build(buildid) + build = self.__wait_and_get_build(buildid) bstatus = build.get_status() logging.info("build_status=%s", bstatus) @@ -159,7 +159,7 @@ def get_result(self, buildid): if bstatus == "UNSTABLE" and \ (build.get_resultset()["skt.cmd_run"].status in ["PASSED", "FIXED"]): - if self.get_baseretcode(buildid) != 0: + if self.__get_baseretcode(buildid) != 0: logging.warning("baseline failure found during patch testing") return sktm.misc.tresult.BASELINE_FAILURE @@ -234,7 +234,7 @@ def build(self, baserepo=None, ref=None, baseconfig=None, self.server.get_job(self.name) expected_id = self.server.get_job(self.name).get_next_build_number() self.server.build_job(self.name, params) - build = self.find_build(params, expected_id) + build = self.__find_build(params, expected_id) logging.info("submitted build: %s", build) return build.get_number() @@ -244,7 +244,7 @@ def is_build_complete(self, buildid): return not build.is_running() - def _params_eq(self, build, params): + def __params_eq(self, build, params): try: build_params = build.get_actions()["parameters"] except (AttributeError, KeyError): @@ -257,7 +257,7 @@ def _params_eq(self, build, params): return True - def find_build(self, params, eid=None): + def __find_build(self, params, eid=None): job = self.server.get_job(self.name) lbuild = None @@ -271,12 +271,12 @@ def find_build(self, params, eid=None): while lbuild.get_number() < eid: time.sleep(1) lbuild = job.get_last_build() - if self._params_eq(lbuild, params): + if self.__params_eq(lbuild, params): return lbuild # slowpath for bid in job.get_build_ids(): build = job.get_build(bid) - if self._params_eq(build, params): + if self.__params_eq(build, params): return build return None From 57b67b6bb18dfa8d9d6bf495781f8d47baa7e171 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 14:20:31 +0300 Subject: [PATCH 5/8] Spell out patch URL list in sktm.jenkins Spell out "patch URL list" instead of writing "patchwork" in sktm.jenkins.Project, to make clear what's being accepted/returned. Leave updating the Jenkins project parameter name for later. --- sktm/__init__.py | 7 ++++--- sktm/jenkins.py | 34 +++++++++++++++++----------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index a4aa4cd..f09d2fa 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -251,6 +251,7 @@ def check_patchwork(self): self.db.set_patchset_pending(cpw.baseurl, cpw.projectid, patchset.get_patch_info_list()) # Submit and remember a Jenkins build for the patchset + url_list = patchset.get_patch_url_list() self.pj.append((sktm.jtype.PATCHWORK, self.jk.build( baserepo=self.baserepo, @@ -259,7 +260,7 @@ def check_patchwork(self): message_id=patchset.message_id, subject=patchset.subject, emails=patchset.email_addr_set, - patchwork=patchset.get_patch_url_list(), + patch_url_list=url_list, makeopts=self.makeopts), cpw)) logging.info("submitted message ID: %s", patchset.message_id) @@ -300,8 +301,8 @@ def check_pending(self): bid ) - patchset = self.jk.get_patchwork(bid) - for purl in patchset: + patch_url_list = self.jk.get_patch_url_list(bid) + for purl in patch_url_list: match = re.match(r"(.*)/patch/(\d+)$", purl) if match: baseurl = match.group(1) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 7ecdf47..e54f42a 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -103,8 +103,7 @@ def get_base_hash(self, buildid): """ return self.__get_cfg_data(buildid, "skt.cmd_merge", "basehead") - # FIXME Clarify function name - def get_patchwork(self, buildid): + def get_patch_url_list(self, buildid): """ Get the list of Patchwork patch URLs for the specified completed build. Wait for the build to complete, if it hasn't yet. @@ -183,24 +182,25 @@ def get_result(self, buildid): # FIXME Clarify/fix argument names def build(self, baserepo=None, ref=None, baseconfig=None, - message_id=None, subject=None, emails=set(), patchwork=[], + message_id=None, subject=None, emails=set(), patch_url_list=[], makeopts=None): """ Submit a build of a patchset. Args: - baserepo: Baseline Git repo URL. - ref: Baseline Git reference to test. - baseconfig: Kernel configuration URL. - message_id: Value of the "Message-Id" header of the e-mail - message representing the patchset, or None if unknown. - subject: Subject of the message representing the patchset, or - None if unknown. - emails: Set of e-mail addresses involved with the patchset to - send notifications to. - patchwork: List of URLs pointing to patches to apply. - makeopts: String of extra arguments to pass to the build's make - invocation. + baserepo: Baseline Git repo URL. + ref: Baseline Git reference to test. + baseconfig: Kernel configuration URL. + message_id: Value of the "Message-Id" header of the e-mail + message representing the patchset, or None if + unknown. + subject: Subject of the message representing the patchset, + or None if unknown. + emails: Set of e-mail addresses involved with the patchset + to send notifications to. + patch_url_list: List of URLs pointing to patches to apply. + makeopts: String of extra arguments to pass to the build's + make invocation. Returns: Submitted build number. @@ -224,8 +224,8 @@ def build(self, baserepo=None, ref=None, baseconfig=None, if emails: params["emails"] = ",".join(emails) - if patchwork: - params["patchwork"] = " ".join(patchwork) + if patch_url_list: + params["patchwork"] = " ".join(patch_url_list) if makeopts is not None: params["makeopts"] = makeopts From 51e357504caf7f318893acac42b808bd777aa17b Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Mon, 28 May 2018 10:20:18 +0300 Subject: [PATCH 6/8] Specify patch order for patch_url_list args --- sktm/jenkins.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index e54f42a..ecc938d 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -112,7 +112,8 @@ def get_patch_url_list(self, buildid): buildid: Jenkins build ID. Return: - The list of Patchwork patch URLs. + The list of Patchwork patch URLs, in the order the patches should + be applied in. """ return self.__get_cfg_data(buildid, "skt.cmd_merge", "pw") @@ -198,7 +199,8 @@ def build(self, baserepo=None, ref=None, baseconfig=None, or None if unknown. emails: Set of e-mail addresses involved with the patchset to send notifications to. - patch_url_list: List of URLs pointing to patches to apply. + patch_url_list: List of URLs pointing to patches to apply, in the + order they should be applied in. makeopts: String of extra arguments to pass to the build's make invocation. From ad52f4f8d3263abacb1d604fb21718b5a8afd601 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 14:34:52 +0300 Subject: [PATCH 7/8] Create Jenkins project interface outside watcher Create a complete Jenkins project interface instance and pass it to the watcher, instead of having the watcher create it. This prepares the watcher to using abstract scheduler interface, instead of Jenkins specifically. --- sktm.py | 8 ++++++-- sktm/__init__.py | 15 +++++---------- tests/test_init.py | 14 +++++++++----- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/sktm.py b/sktm.py index d90ce8d..126a185 100755 --- a/sktm.py +++ b/sktm.py @@ -19,6 +19,7 @@ import logging import os import sktm +import sktm.jenkins def setup_parser(): @@ -122,9 +123,12 @@ def load_config(args): setup_logging(args.verbose) cfg = load_config(args) logging.debug("cfg=%s", cfg) + jenkins_project = sktm.jenkins.JenkinsProject(cfg.get("jjname"), + cfg.get("jurl"), + cfg.get("jlogin"), + cfg.get("jpass")) - sw = sktm.watcher(cfg.get("jurl"), cfg.get("jlogin"), cfg.get("jpass"), - cfg.get("jjname"), cfg.get("db"), + sw = sktm.watcher(jenkins_project, cfg.get("db"), cfg.get("filter"), cfg.get("makeopts")) args.func(sw, cfg) diff --git a/sktm/__init__.py b/sktm/__init__.py index f09d2fa..3409a09 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -32,16 +32,13 @@ class jtype(enum.IntEnum): # TODO This is no longer just a watcher. Rename/refactor/describe accordingly. class watcher(object): - def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, - jenkinsjobname, dbpath, filter, makeopts=None): + def __init__(self, jenkins_project, dbpath, filter, makeopts=None): """ Initialize a "watcher". Args: - jenkinsurl: Jenkins instance URL. - jenkinslogin: Jenkins user name. - jenkinspassword: Jenkins user password. - jenkinsjobname: Name of the Jenkins job to trigger and watch. + jenkins_project: Interface to the Jenkins project to trigger + and watch (sktm.jenkins.JenkinsProject). dbpath: Path to the job status database file. filter: The name of a patchset filter program. The program should accept a list of mbox URLs @@ -56,12 +53,10 @@ def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, building. """ # FIXME Clarify/fix member variable names + # Jenkins project interface instance + self.jk = jenkins_project # Database instance self.db = sktm.db.skt_db(os.path.expanduser(dbpath)) - # Jenkins project interface instance - self.jk = sktm.jenkins.JenkinsProject(jenkinsjobname, - jenkinsurl, jenkinslogin, - jenkinspassword) # Patchset filter program self.filter = filter # Extra arguments to pass to "make" diff --git a/tests/test_init.py b/tests/test_init.py index 666c3f6..26913b8 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -24,17 +24,21 @@ class TestInit(unittest.TestCase): """Test cases for the __init__ module.""" - @mock.patch('sktm.jenkins.skt_jenkins', Mock()) + @mock.patch('sktm.jenkins.JenkinsProject', Mock()) def setUp(self): """Test fixtures for testing __init__.""" self.database_dir = tempfile.mkdtemp() self.database_file = "{}/testdb.sqlite".format(self.database_dir) + jenkins_project = sktm.jenkins.JenkinsProject( + name="sktm_jenkins_job", + url="http://example.com/jenkins", + username="username", + password="password" + ) + self.watcher_obj = sktm.watcher( - jenkinsurl="http://example.com/jenkins", - jenkinslogin="username", - jenkinspassword="password", - jenkinsjobname="sktm_jenkins_job", + jenkins_project, dbpath=self.database_file, filter=None, makeopts=None From c5b64b095797b78ae4cf081cc6eba1ed647b0a06 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 24 May 2018 14:46:18 +0300 Subject: [PATCH 8/8] Document sktm.jenkins.Project.is_build_complete() --- sktm/jenkins.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index ecc938d..5212633 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -241,6 +241,15 @@ def build(self, baserepo=None, ref=None, baseconfig=None, return build.get_number() def is_build_complete(self, buildid): + """ + Check if a project build is complete. + + Args: + buildid: Jenkins build ID to get the status of. + + Return: + True if the build is complete, False if not. + """ job = self.server.get_job(self.name) build = job.get_build(buildid)