From 37e2ce2024c1a606b7864109ceced7acdb3b8238 Mon Sep 17 00:00:00 2001 From: danghai Date: Fri, 13 Jul 2018 14:13:22 -0700 Subject: [PATCH 1/7] Rename class tresult to TestResult, and move into sktm.misc Move sktm.tresult to sktm.misc.TestResult to fix composition, so that modules lower in the hierarchy (e.g. sktm.jenkins) do not have to import the module above (i.e. sktm). --- sktm/__init__.py | 12 ++---------- sktm/db.py | 10 +++++----- sktm/jenkins.py | 14 +++++++------- sktm/misc.py | 24 ++++++++++++++++++++++++ sktm/patchwork.py | 20 ++++++++++---------- 5 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 sktm/misc.py diff --git a/sktm/__init__.py b/sktm/__init__.py index b16931c..f3e01ce 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -22,6 +22,7 @@ import sktm.db import sktm.jenkins +import sktm.misc import sktm.patchwork @@ -43,15 +44,6 @@ def join_with_slash(base, *suffix): return '/'.join(parts) + ending -class tresult(enum.IntEnum): - """Test result""" - ERROR = -1 - SUCCESS = 0 - MERGE_FAILURE = 1 - BUILD_FAILURE = 2 - TEST_FAILURE = 4 - - class jtype(enum.IntEnum): """Job type""" BASELINE = 0 @@ -360,7 +352,7 @@ def check_pending(self): pjt, bid, bres.name, rurl) self.pj.remove((pjt, bid, cpw)) - if bres == sktm.tresult.ERROR: + if bres == sktm.misc.TestResult.ERROR: logging.warning("job completed with an error, ignoring") continue diff --git a/sktm/db.py b/sktm/db.py index 5cb5d22..4b95986 100644 --- a/sktm/db.py +++ b/sktm/db.py @@ -17,7 +17,7 @@ import os import sqlite3 import time -import sktm +import sktm.misc class SktDb(object): @@ -321,7 +321,7 @@ def __get_baselineresult(self, baserepo, commithash): if not result: return None - return sktm.tresult(result[0]) + return sktm.misc.TestResult(result[0]) def get_stable(self, baserepo): """Get the latest stable commit ID for a baseline Git repo URL. @@ -434,8 +434,8 @@ def update_baseline(self, baserepo, commithash, commitdate, baserepo: Baseline Git repo URL. commithash: Commit SHA of the baseline commit. commitdate: Date of the commit. - result: Result ID of the test run (an sktm.tresult). - Cannot be sktm.tresult.ERROR. + result: Result ID of the test run (an sktm.misc.TestResult). + Cannot be sktm.misc.TestResult.ERROR. build_id: The build ID of the test run. """ @@ -550,7 +550,7 @@ def dump_baseline_tests(self): # pragma: no cover for (burl, commit, res, buildid) in self.cur.fetchall(): print("repo url:", burl) print("commit id:", commit) - print("result:", sktm.tresult(res).name) + print("result:", sktm.misc.TestResult(res).name) print("build id: #", buildid, sep='') print("---") diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 74ddc1b..12d7286 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -18,7 +18,7 @@ import jenkinsapi -import sktm +import sktm.misc class skt_jenkins(object): @@ -234,7 +234,7 @@ def get_result(self, jobname, buildid): buildid: Jenkins build ID. Return: - Status of the build (an sktm.tresult). + Status of the build (an sktm.misc.TestResult). """ build = self._wait_and_get_build(jobname, buildid) @@ -242,13 +242,13 @@ def get_result(self, jobname, buildid): logging.info("build_status=%s", bstatus) if bstatus == "SUCCESS": - return sktm.tresult.SUCCESS + return sktm.misc.TestResult.SUCCESS elif bstatus == "UNSTABLE": # Find earliest (worst) step failure step_failure_result_list = [ - ("skt.cmd_merge", sktm.tresult.MERGE_FAILURE), - ("skt.cmd_build", sktm.tresult.BUILD_FAILURE), - ("skt.cmd_run", sktm.tresult.TEST_FAILURE), + ("skt.cmd_merge", sktm.misc.TestResult.MERGE_FAILURE), + ("skt.cmd_build", sktm.misc.TestResult.BUILD_FAILURE), + ("skt.cmd_run", sktm.misc.TestResult.TEST_FAILURE), ] for (step, failure_result) in step_failure_result_list: if set(self.__get_data_list(jobname, buildid, @@ -260,7 +260,7 @@ def get_result(self, jobname, buildid): bstatus) else: logging.warning("Reporting build status \"%s\" as error", bstatus) - return sktm.tresult.ERROR + return sktm.misc.TestResult.ERROR # FIXME Clarify/fix argument names def build(self, jobname, baserepo=None, ref=None, baseconfig=None, diff --git a/sktm/misc.py b/sktm/misc.py new file mode 100644 index 0000000..4719b3f --- /dev/null +++ b/sktm/misc.py @@ -0,0 +1,24 @@ +# Copyright (c) 2017-2018 Red Hat, Inc. All rights reserved. This copyrighted +# material is made available to anyone wishing to use, modify, copy, or +# redistribute it subject to the terms and conditions of the GNU General +# Public License v.2 or later. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import enum + + +class TestResult(enum.IntEnum): + """Test result""" + ERROR = -1 + SUCCESS = 0 + MERGE_FAILURE = 1 + BUILD_FAILURE = 2 + TEST_FAILURE = 4 diff --git a/sktm/patchwork.py b/sktm/patchwork.py index dfe0772..a590ac1 100644 --- a/sktm/patchwork.py +++ b/sktm/patchwork.py @@ -25,7 +25,7 @@ import dateutil.parser import requests -import sktm +import sktm.misc SKIP_PATTERNS = [ @@ -658,14 +658,14 @@ def __set_patch_check(self, patch, payload): def set_patch_check(self, pid, jurl, result): """ Add a patch "check" for the specified patch, with the specified - Jenkins build URL and result (sktm.tresult). The result cannot be - sktm.tresult.ERROR. + Jenkins build URL and result (sktm.misc.TestResult). The result + cannot be sktm.misc.TestResult.ERROR. Args: pid: The ID of the patch to add the "check" for. jurl: Jenkins build URL for the "check" to reference. - result: Test result (sktm.tresult) to feature in the "check" - state. + result: Test result (sktm.misc.TestResult) to feature + in the "check" state. """ if self.apikey is None: logging.debug("No patchwork api key provided, not setting checks") @@ -676,7 +676,7 @@ def set_patch_check(self, pid, jurl, result): 'target_url': jurl, 'context': 'Kernel CI', 'description': 'Kernel CI testing'} - if result == sktm.tresult.SUCCESS: + if result == sktm.misc.TestResult.SUCCESS: payload['state'] = PW_CHECK_CHOICES['success'] else: payload['state'] = PW_CHECK_CHOICES['fail'] @@ -988,14 +988,14 @@ def __get_patch_list(self, filt): def set_patch_check(self, pid, jurl, result): """ Add a patch "check" for the specified patch, with the specified - Jenkins build URL and result (sktm.tresult). The result cannot be - sktm.tresult.ERROR. + Jenkins build URL and result (sktm.misc.TestResult). The result + cannot be sktm.misc.TestResult.ERROR. Args: pid: The ID of the patch to add the "check" for. jurl: Jenkins build URL for the "check" to reference. - result: Test result (sktm.tresult) to feature in the "check" - state. + result: Test result (sktm.misc.TestResult) to feature + in the "check" state. """ # TODO: Implement this for xmlrpc pass From 67a4c0081026abe126438a4cf9ec80c873952941 Mon Sep 17 00:00:00 2001 From: danghai Date: Fri, 13 Jul 2018 21:56:37 -0700 Subject: [PATCH 2/7] Convert Jenkins interface to Jenkins project interface Instead of creating and using a "Jenkins Interface" (sktm.jenkins.JenkinsProject), and then supplying a project name ("jobname") with every method call, create and use a "Jenkins Project Interface" (sktm.jenkins.JenkinsProject) and only supply the project name on its creation. This simplifes the interface, and removes a bit of code. This also prepares for abstracting Jenkins away. --- sktm/__init__.py | 24 +++++----- sktm/jenkins.py | 117 ++++++++++++++++++++------------------------- tests/test_init.py | 2 +- 3 files changed, 63 insertions(+), 80 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index f3e01ce..bf6300f 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -79,10 +79,10 @@ def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, # Database instance self.db = sktm.db.SktDb(os.path.expanduser(dbpath)) # Jenkins interface instance - self.jk = sktm.jenkins.skt_jenkins(jenkinsurl, jenkinslogin, - jenkinspassword) - # Jenkins project name - self.jobname = jenkinsjobname + self.jk = sktm.jenkins.JenkinsProject(jenkinsjobname, + jenkinsurl, + jenkinslogin, + jenkinspassword) # Patchset filter program self.patch_filter = patch_filter # Extra arguments to pass to "make" @@ -190,8 +190,7 @@ def add_pw(self, baseurl, pname, lpatch=None, apikey=None, skip=[]): def check_baseline(self): """Submit a build for baseline""" self.pj.append((sktm.jtype.BASELINE, - self.jk.build(self.jobname, - baserepo=self.baserepo, + self.jk.build(baserepo=self.baserepo, ref=self.baseref, baseconfig=self.cfgurl, makeopts=self.makeopts), @@ -323,7 +322,6 @@ def check_patchwork(self): # Submit and remember a Jenkins build for the series self.pj.append((sktm.jtype.PATCHWORK, self.jk.build( - self.jobname, baserepo=self.baserepo, ref=stablecommit, baseconfig=self.cfgurl, @@ -341,11 +339,11 @@ def check_patchwork(self): def check_pending(self): for (pjt, bid, cpw) in self.pj: - if self.jk.is_build_complete(self.jobname, bid): - bres = self.jk.get_result(self.jobname, bid) - rurl = self.jk.get_result_url(self.jobname, bid) - basehash = self.jk.get_base_hash(self.jobname, bid) - basedate = self.jk.get_base_commitdate(self.jobname, bid) + if self.jk.is_build_complete(bid): + bres = self.jk.get_result(bid) + rurl = self.jk.get_result_url(bid) + basehash = self.jk.get_base_hash(bid) + basedate = self.jk.get_base_commitdate(bid) logging.info("job completed: " "type=%d; jjid=%d; result=%s; url=%s", @@ -367,7 +365,7 @@ def check_pending(self): elif pjt == sktm.jtype.PATCHWORK: patches = list() - patch_url_list = self.jk.get_patchwork(self.jobname, bid) + patch_url_list = self.jk.get_patchwork(bid) for patch_url in patch_url_list: patches.append(self.get_patch_info_from_url(cpw, patch_url)) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 12d7286..d88539c 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -21,22 +21,25 @@ import sktm.misc -class skt_jenkins(object): - """Jenkins interface""" - def __init__(self, url, username=None, password=None): +class JenkinsProject(object): + """Jenkins project interface""" + def __init__(self, name, url, username=None, password=None): """ - Initialize a Jenkins interface. + Initialize a Jenkins project interface. Args: + name: Name of the Jenkins project to operate on. url: Jenkins instance URL. username: Jenkins user name. password: Jenkins user password. """ + self.name = name + # Initialize Jenkins server interface # TODO Add support for CSRF protection self.server = jenkinsapi.jenkins.Jenkins(url, username, password) - def _wait_and_get_build(self, jobname, buildid): - job = self.server.get_job(jobname) + def _wait_and_get_build(self, buildid): + job = self.server.get_job(self.name) build = job.get_build(buildid) build.block_until_complete(delay=60) @@ -45,14 +48,13 @@ def _wait_and_get_build(self, jobname, buildid): return build - def __get_data_list(self, jobname, buildid, stepname, key): + def __get_data_list(self, buildid, stepname, key): """ Get a list of values of a build resultset key, for all steps matching - the specified name, of the specified completed build, for the - specified project. Wait for the build to complete, if it hasn't yet. + the specified name, of the specified completed build. Wait for the + build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. stepname: A path matching test steps in the result, which resultset should be accessed. @@ -62,7 +64,7 @@ def __get_data_list(self, jobname, buildid, stepname, key): The list of key values. """ value_list = [] - build = self._wait_and_get_build(jobname, buildid) + build = self._wait_and_get_build(buildid) if not build.has_resultset(): raise Exception("No results for build %d (%s)" % @@ -74,16 +76,14 @@ def __get_data_list(self, jobname, buildid, stepname, key): return value_list - def __get_cfg_data_list(self, jobname, buildid, stepname, + def __get_cfg_data_list(self, buildid, stepname, cfgkey, default=None): """ Get a list of values from a JSON-formatted output of a test result, for all steps matching the specified name, of the specified completed - build for the specified project. Wait for the build to complete, if it - hasn't yet. + build. Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. stepname: A path matching test steps in the result, which output should be parsed as JSON. @@ -97,25 +97,22 @@ def __get_cfg_data_list(self, jobname, buildid, stepname, not found. """ value_list = [] - for stdout in self.__get_data_list(jobname, buildid, - stepname, "stdout"): + for stdout in self.__get_data_list(buildid, stepname, "stdout"): logging.debug("stdout=%s", stdout) cfg = json.loads(stdout) value_list.append(cfg.get(cfgkey, default)) return value_list - def __get_cfg_data_uniform(self, jobname, buildid, stepname, + def __get_cfg_data_uniform(self, buildid, stepname, cfgkey, default=None): """ Get a uniform value from a JSON-formatted output of a test result, for all steps matching the specified name, of the specified completed - build for the specified project. Wait for the build to complete, if it - hasn't yet. Throw an exception if the value is not uniform across all - steps. + build. Wait for the build to complete, if it hasn't yet. Throw an + exception if the value is not uniform across all steps. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. stepname: A path matching test steps in the result, which output should be parsed as JSON. @@ -134,19 +131,17 @@ def verify(x, y): return x return reduce(verify, - self.__get_cfg_data_list(jobname, buildid, stepname, + self.__get_cfg_data_list(buildid, stepname, cfgkey, default)) - def __get_cfg_data_max(self, jobname, buildid, stepname, + def __get_cfg_data_max(self, buildid, stepname, cfgkey, default=None): """ Get the maximum value from a JSON-formatted output of a test result, for all steps matching the specified name, of the specified completed - build for the specified project. Wait for the build to complete, if it - hasn't yet. + build. Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. stepname: A path matching test steps in the result, which output should be parsed as JSON. @@ -159,84 +154,76 @@ def __get_cfg_data_max(self, jobname, buildid, stepname, The maximum value of the key across all steps. """ return reduce(lambda x, y: (x if x > y else y), - self.__get_cfg_data_list(jobname, buildid, stepname, + self.__get_cfg_data_list(buildid, stepname, cfgkey, default)) - def get_base_commitdate(self, jobname, buildid): + def get_base_commitdate(self, buildid): """ - Get base commit's committer date of the specified completed build for - the specified project. Wait for the build to complete, if it hasn't - yet. + Get base commit's committer date of the specified completed build. + Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The epoch timestamp string of the committer date. """ - return self.__get_cfg_data_uniform(jobname, buildid, "skt.cmd_merge", + return self.__get_cfg_data_uniform(buildid, "skt.cmd_merge", "commitdate") - def get_base_hash(self, jobname, buildid): + def get_base_hash(self, buildid): """ - Get base commit's hash of the specified completed build for the - specified project. Wait for the build to complete, if it hasn't yet. + Get base commit's hash of the specified completed build. + Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The base commit's hash string. """ - return self.__get_cfg_data_uniform(jobname, buildid, "skt.cmd_merge", + return self.__get_cfg_data_uniform(buildid, "skt.cmd_merge", "basehead") # FIXME Clarify function name - def get_patchwork(self, jobname, buildid): + def get_patchwork(self, buildid): """ - Get the list of Patchwork patch URLs for the specified completed build - for the specified project. Wait for the build to complete, if it - hasn't yet. + Get the list of Patchwork patch URLs for the specified completed + build. Wait for the build to complete, if it hasn't yet. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: The list of Patchwork patch URLs. """ - return self.__get_cfg_data_uniform(jobname, buildid, "skt.cmd_merge", - "pw") + return self.__get_cfg_data_uniform(buildid, "skt.cmd_merge", "pw") - def get_baseretcode(self, jobname, buildid): + def get_baseretcode(self, buildid): """ Get the maximum (the worst) return code of a baseline test across all "run" steps for the specified completed build of the specified project. Wait for the build to complete, if it hasn't yet. """ - return self.__get_cfg_data_max(jobname, buildid, "skt.cmd_run", + return self.__get_cfg_data_max(buildid, "skt.cmd_run", "baseretcode", 0) - def get_result_url(self, jobname, buildid): + def get_result_url(self, buildid): return sktm.join_with_slash(self.server.base_server_url(), "job", - jobname, str(buildid)) - def get_result(self, jobname, buildid): + def get_result(self, buildid): """ - Get the status of a build for specified project name and build ID. + Get the status of a build for specified build ID. Args: - jobname: Jenkins project name. buildid: Jenkins build ID. Return: Status of the build (an sktm.misc.TestResult). """ - build = self._wait_and_get_build(jobname, buildid) + build = self._wait_and_get_build(buildid) bstatus = build.get_status() logging.info("build_status=%s", bstatus) @@ -251,8 +238,7 @@ def get_result(self, jobname, buildid): ("skt.cmd_run", sktm.misc.TestResult.TEST_FAILURE), ] for (step, failure_result) in step_failure_result_list: - if set(self.__get_data_list(jobname, buildid, - step, "status")) & \ + if set(self.__get_data_list(buildid, step, "status")) & \ set(["FAILED", "REGRESSION"]): return failure_result logging.warning("Build status is \"%s\", " @@ -263,14 +249,13 @@ def get_result(self, jobname, buildid): return sktm.misc.TestResult.ERROR # FIXME Clarify/fix argument names - def build(self, jobname, baserepo=None, ref=None, baseconfig=None, + def build(self, baserepo=None, ref=None, baseconfig=None, message_id=None, subject=None, emails=set(), patchwork=[], makeopts=None): """ Submit a build of a patch series. Args: - jobname: Name of the Jenkins project to build. baserepo: Baseline Git repo URL. ref: Baseline Git reference to test. baseconfig: Kernel configuration URL. @@ -313,15 +298,15 @@ def build(self, jobname, baserepo=None, ref=None, baseconfig=None, params["makeopts"] = makeopts logging.debug(params) - self.server.get_job(jobname) - expected_id = self.server.get_job(jobname).get_next_build_number() - self.server.build_job(jobname, params) - build = self.find_build(jobname, params, expected_id) + self.server.get_job(self.name) + expected_id = self.server.get_job(self.name).get_next_build_number() + self.server.build_job(self.name, params) + build = self.find_build(params, expected_id) logging.info("submitted build: %s", build) return build.get_number() - def is_build_complete(self, jobname, buildid): - job = self.server.get_job(jobname) + def is_build_complete(self, buildid): + job = self.server.get_job(self.name) build = job.get_build(buildid) return not build.is_running() @@ -339,8 +324,8 @@ def _params_eq(self, build, params): return True - def find_build(self, jobname, params, eid=None): - job = self.server.get_job(jobname) + def find_build(self, params, eid=None): + job = self.server.get_job(self.name) lbuild = None while not lbuild: diff --git a/tests/test_init.py b/tests/test_init.py index 55b222b..4ffc4ac 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -56,7 +56,7 @@ def test_join_with_slash(self): class TestInit(unittest.TestCase): """Test cases for the __init__ module.""" - @mock.patch('sktm.jenkins.skt_jenkins', Mock()) + @mock.patch('sktm.jenkins.JenkinsProject', Mock()) def setUp(self): """Test fixtures for testing __init__.""" self.database_dir = tempfile.mkdtemp() From 94adc08af6a58b83905cedb21f2a6a1813bf41fa Mon Sep 17 00:00:00 2001 From: danghai Date: Fri, 13 Jul 2018 22:55:03 -0700 Subject: [PATCH 3/7] Spell out patch URL list in sktm.jenkins Spell out "patch URL list" instead of writing "patchwork" in sktm.jenkins.JenkinsProject, to make clear what's being accepted/returned. Leave updating the Jenkins project parameter name for later. --- sktm/__init__.py | 8 ++++---- sktm/jenkins.py | 34 +++++++++++++++++----------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index bf6300f..86c5d14 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -320,6 +320,7 @@ def check_patchwork(self): self.db.set_patchset_pending(cpw.baseurl, cpw.project_id, series.get_patch_info_list()) # Submit and remember a Jenkins build for the series + url_list = series.get_patch_url_list() self.pj.append((sktm.jtype.PATCHWORK, self.jk.build( baserepo=self.baserepo, @@ -328,14 +329,13 @@ def check_patchwork(self): message_id=series.message_id, subject=series.subject, emails=series.email_addr_set, - patchwork=series.get_patch_url_list(), + patch_url_list=url_list, makeopts=self.makeopts), cpw)) logging.info("submitted message ID: %s", series.message_id) logging.info("submitted subject: %s", series.subject) logging.info("submitted emails: %s", series.email_addr_set) - logging.info("submitted series: %s", - series.get_patch_url_list()) + logging.info("submitted series: %s", url_list) def check_pending(self): for (pjt, bid, cpw) in self.pj: @@ -365,7 +365,7 @@ def check_pending(self): elif pjt == sktm.jtype.PATCHWORK: patches = list() - patch_url_list = self.jk.get_patchwork(bid) + patch_url_list = self.jk.get_patch_url_list(bid) for patch_url in patch_url_list: patches.append(self.get_patch_info_from_url(cpw, patch_url)) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index d88539c..2d5ae10 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -185,8 +185,7 @@ def get_base_hash(self, buildid): return self.__get_cfg_data_uniform(buildid, "skt.cmd_merge", "basehead") - # FIXME Clarify function name - def get_patchwork(self, buildid): + def get_patch_url_list(self, buildid): """ Get the list of Patchwork patch URLs for the specified completed build. Wait for the build to complete, if it hasn't yet. @@ -250,24 +249,25 @@ def get_result(self, buildid): # FIXME Clarify/fix argument names def build(self, baserepo=None, ref=None, baseconfig=None, - message_id=None, subject=None, emails=set(), patchwork=[], + message_id=None, subject=None, emails=set(), patch_url_list=[], makeopts=None): """ Submit a build of a patch series. Args: - baserepo: Baseline Git repo URL. - ref: Baseline Git reference to test. - baseconfig: Kernel configuration URL. - message_id: Value of the "Message-Id" header of the e-mail - message representing the series, or None if unknown. - subject: Subject of the message representing the series, or - None if unknown. - emails: Set of e-mail addresses involved with the series to - send notifications to. - patchwork: List of URLs pointing to patches to apply. - makeopts: String of extra arguments to pass to the build's make - invocation. + baserepo: Baseline Git repo URL. + ref: Baseline Git reference to test. + baseconfig: Kernel configuration URL. + message_id: Value of the "Message-Id" header of the e-mail + message representing the series, or None if + unknown. + subject: Subject of the message representing the series, + or None if unknown. + emails: Set of e-mail addresses involved with the series + to send notifications to. + patch_url_list: List of URLs pointing to patches to apply. + makeopts: String of extra arguments to pass to the build's + make invocation. Returns: Submitted build number. @@ -291,8 +291,8 @@ def build(self, baserepo=None, ref=None, baseconfig=None, if emails: params["emails"] = ",".join(emails) - if patchwork: - params["patchwork"] = " ".join(patchwork) + if patch_url_list: + params["patchwork"] = " ".join(patch_url_list) if makeopts is not None: params["makeopts"] = makeopts From 6a92abac1fbdef57726f6faa310f9986405b97be Mon Sep 17 00:00:00 2001 From: danghai Date: Sat, 14 Jul 2018 01:00:51 -0700 Subject: [PATCH 4/7] Specify patch order for patch_url_list args --- sktm/jenkins.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 2d5ae10..e578d73 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -194,7 +194,8 @@ def get_patch_url_list(self, buildid): buildid: Jenkins build ID. Return: - The list of Patchwork patch URLs. + The list of Patchwork patch URLs, in the order the patches should + be applied in. """ return self.__get_cfg_data_uniform(buildid, "skt.cmd_merge", "pw") @@ -265,7 +266,8 @@ def build(self, baserepo=None, ref=None, baseconfig=None, or None if unknown. emails: Set of e-mail addresses involved with the series to send notifications to. - patch_url_list: List of URLs pointing to patches to apply. + patch_url_list: List of URLs pointing to patches to apply, in the + order they should be applied in. makeopts: String of extra arguments to pass to the build's make invocation. From 0e55b3e500380f7212b36aafb89b12c754cb9ed6 Mon Sep 17 00:00:00 2001 From: danghai Date: Sat, 14 Jul 2018 01:04:10 -0700 Subject: [PATCH 5/7] Document sktm.jenkins.Project.is_build_complete() --- sktm/jenkins.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index e578d73..88f92c5 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -308,6 +308,15 @@ def build(self, baserepo=None, ref=None, baseconfig=None, return build.get_number() def is_build_complete(self, buildid): + """ + Check if a build is complete. + + Args: + buildid: Jenkins build ID to get the status of. + + Return: + True if the build is complete, False if not. + """ job = self.server.get_job(self.name) build = job.get_build(buildid) From d9b6c0e25317ae1a9674ac54295776050016396d Mon Sep 17 00:00:00 2001 From: danghai Date: Sat, 14 Jul 2018 12:30:44 -0700 Subject: [PATCH 6/7] Create Jenkins project interface outside watcher Create a complete Jenkins project interface instance and pass it to the watcher, instead of having the watcher create it. This prepares the watcher to using abstract scheduler interface, instead of Jenkins specifically. --- sktm/__init__.py | 13 +++---------- sktm/executable.py | 9 +++++++-- tests/test_init.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/sktm/__init__.py b/sktm/__init__.py index 86c5d14..0636035 100644 --- a/sktm/__init__.py +++ b/sktm/__init__.py @@ -24,6 +24,7 @@ import sktm.jenkins import sktm.misc import sktm.patchwork +import sktm.jenkins def join_with_slash(base, *suffix): @@ -52,16 +53,11 @@ class jtype(enum.IntEnum): # TODO This is no longer just a watcher. Rename/refactor/describe accordingly. class watcher(object): - def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, - jenkinsjobname, dbpath, patch_filter, makeopts=None): + def __init__(self, jenkins_project, dbpath, patch_filter, makeopts=None): """ Initialize a "watcher". Args: - jenkinsurl: Jenkins instance URL. - jenkinslogin: Jenkins user name. - jenkinspassword: Jenkins user password. - jenkinsjobname: Name of the Jenkins job to trigger and watch. dbpath: Path to the job status database file. patch_filter: The name of a patch series filter program. The program should accept a list of mbox URLs @@ -79,10 +75,7 @@ def __init__(self, jenkinsurl, jenkinslogin, jenkinspassword, # Database instance self.db = sktm.db.SktDb(os.path.expanduser(dbpath)) # Jenkins interface instance - self.jk = sktm.jenkins.JenkinsProject(jenkinsjobname, - jenkinsurl, - jenkinslogin, - jenkinspassword) + self.jk = jenkins_project # Patchset filter program self.patch_filter = patch_filter # Extra arguments to pass to "make" diff --git a/sktm/executable.py b/sktm/executable.py index 4aa9700..78ec51b 100644 --- a/sktm/executable.py +++ b/sktm/executable.py @@ -19,6 +19,7 @@ import logging import os import sktm +import sktm.jenkins def setup_parser(): @@ -129,8 +130,12 @@ def main(): cfg = load_config(args) logging.debug("cfg=%s", cfg) - sw = sktm.watcher(cfg.get("jurl"), cfg.get("jlogin"), cfg.get("jpass"), - cfg.get("jjname"), cfg.get("db"), + jenkins_project = sktm.jenkins.JenkinsProject(cfg.get("jjname"), + cfg.get("jurl"), + cfg.get("jlogin"), + cfg.get("jpass")) + + sw = sktm.watcher(jenkins_project, cfg.get("db"), cfg.get("filter"), cfg.get("makeopts")) args.func(sw, cfg) diff --git a/tests/test_init.py b/tests/test_init.py index 4ffc4ac..a91267d 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -62,11 +62,15 @@ def setUp(self): self.database_dir = tempfile.mkdtemp() self.database_file = "{}/testdb.sqlite".format(self.database_dir) + jenkins_project = sktm.jenkins.JenkinsProject( + name="sktm_jenkins_job", + url="http://example.com/jenkins", + username="username", + password="password" + ) + self.watcher_obj = sktm.watcher( - jenkinsurl="http://example.com/jenkins", - jenkinslogin="username", - jenkinspassword="password", - jenkinsjobname="sktm_jenkins_job", + jenkins_project, dbpath=self.database_file, patch_filter=None, makeopts=None From 390543af7ce30c9909ce87ae81b93c0c2afd842c Mon Sep 17 00:00:00 2001 From: danghai Date: Thu, 19 Jul 2018 02:16:59 -0700 Subject: [PATCH 7/7] Describe sktm.jenkins.get_result* functions --- sktm/jenkins.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sktm/jenkins.py b/sktm/jenkins.py index 88f92c5..fda9b68 100644 --- a/sktm/jenkins.py +++ b/sktm/jenkins.py @@ -209,19 +209,32 @@ def get_baseretcode(self, buildid): "baseretcode", 0) def get_result_url(self, buildid): + """ + Get the URL of the web representation of the specified build of the + specified Jenkins project. + + Args: + jobname: Jenkins project name. + buildid: Jenkins build ID. + + Result: + The URL of the build result. + """ return sktm.join_with_slash(self.server.base_server_url(), "job", str(buildid)) def get_result(self, buildid): """ - Get the status of a build for specified build ID. + Get result code (sktm.misc.TestResult) for the specified build of the + specified Jenkins project. Wait for the build to complete, if it + hasn't yet. Args: buildid: Jenkins build ID. Return: - Status of the build (an sktm.misc.TestResult). + The build result code (sktm.misc.TestResult). """ build = self._wait_and_get_build(buildid)