diff --git a/pytest_cafy/plugin.py b/pytest_cafy/plugin.py
index 74ff9c3..d65d1e8 100755
--- a/pytest_cafy/plugin.py
+++ b/pytest_cafy/plugin.py
@@ -19,13 +19,15 @@
import inspect
import yaml
import pytest
+import traceback
from _pytest.terminal import TerminalReporter
-from _pytest.runner import runtestprotocol
+from _pytest.runner import runtestprotocol, TestReport
from _pytest.mark import MarkInfo
from enum import Enum
from tabulate import tabulate
+from pprint import pprint, pformat
from shutil import copyfile
from configparser import ConfigParser
from datetime import datetime
@@ -35,12 +37,18 @@
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from jinja2 import Template
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
from logger.cafylog import CafyLog
from topology.topo_mgr.topo_mgr import Topology
from utils.cafyexception import CafyException
from debug import DebugLibrary
+from wrapt_timeout_decorator import timeout as wrapt_timeout
+
+
#Check with CAFYKIT_HOME or GIT_REPO or CAFYAP_REPO environment is set,
#if all are set, CAFYAP_REPO takes precedence
CAFY_REPO = os.environ.get("CAFYAP_REPO", None)
@@ -122,7 +130,7 @@ def pytest_addoption(parser):
group.addoption('--work-dir', dest="workdir", metavar="DIR", default=None,
help="Path for work dir")
- group.addoption('--report-dir', dest="reportdir",
+ group.addoption('-R','--report-dir', dest="reportdir",
metavar="DIR",
default=None,
help="Path for report dir")
@@ -133,7 +141,7 @@ def pytest_addoption(parser):
type=lambda x: is_valid_param(x, file_type='topology_file'),
help='Filename of your testbed')
- group.addoption('--test-input-file', action='store', dest='test_input_file',
+ group.addoption('-I', '--test-input-file', action='store', dest='test_input_file',
metavar='test_input_file',
type=lambda x: is_valid_param(x, file_type='input_file'),
help='Filename of your test input file')
@@ -534,7 +542,7 @@ def pytest_collection_modifyitems(session, config, items):
log.info("url: {}".format(url))
log.info("Calling API service for live logging of reg_id ")
params = {"reg_id": CafyLog.registration_id}
- response = requests.patch(url, json=params, headers=headers)
+ response = requests.patch(url, json=params, headers=headers, timeout=120)
if response.status_code == 200:
log.info("Calling API service for live logging of reg_id successful")
else:
@@ -558,7 +566,7 @@ def pytest_collection_modifyitems(session, config, items):
url = '{0}/api/runs/{1}/cases'.format(os.environ.get('CAFY_API_HOST'), os.environ.get('CAFY_RUN_ID'))
log.info("url: {}".format(url))
log.info("Calling API service for live logging of collected testcases ")
- response = requests.post(url, json=CafyLog.collected_testcases, headers=headers)
+ response = requests.post(url, json=CafyLog.collected_testcases, headers=headers, timeout=120)
if response.status_code == 200:
log.info("Calling API service for live logging of collected testcases successful")
else:
@@ -675,7 +683,7 @@ def _sendemail(self):
msg['To'] = mail_to
msg.add_header('Content-Type', 'text/html')
# fixme: add an option to read config from file rather then CLI
- with smtplib.SMTP(self.smtp_server, self.smtp_port) as mail_server:
+ with smtplib.SMTP(self.smtp_server, self.smtp_port,timeout=60) as mail_server:
if self.email_from_passwd:
mail_server.ehlo()
mail_server.starttls()
@@ -789,7 +797,7 @@ def initiate_analyzer(self, reg_id, test_case, debug_server):
try:
url = "http://{0}:5001/initiate_analyzer/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to initialize analyzer" % url)
- response = requests.post(url, data=params)
+ response = requests.post(url, data=params, timeout=300)
if response.status_code == 200:
self.log.info("Analyzer initialized")
return True
@@ -813,7 +821,7 @@ def pytest_runtest_teardown(self, item, nextitem):
self.log.set_testcase("Teardown")
else:
testcase_name = self.get_test_name(nextitem.nodeid)
- self.log.set_testcase(testcase_name)
+ # self.log.set_testcase(testcase_name)
testcase_name = self.get_test_name(item.nodeid)
self.log.info('Teardown module for testcase {}'.format(testcase_name))
@@ -863,7 +871,7 @@ def check_analyzer_status(self, params, headers):
try:
url = "http://{0}:5001/end_test_case/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to check analyzer status" % url)
- response = requests.get(url, data=params)
+ response = requests.get(url, data=params, timeout=60)
if response.status_code == 200:
return response.json()['analyzer_status']
else:
@@ -894,7 +902,7 @@ def pytest_runtest_logreport(self, report):
try:
url = 'http://{0}:5001/registertest/'.format(CafyLog.debug_server)
self.log.info("Calling registration service to start handshake(url:%s" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
self.log.info("Handshake part of registration service was successful")
else:
@@ -905,14 +913,6 @@ def pytest_runtest_logreport(self, report):
if report.when == 'teardown':
- if self.reg_dict:
- reg_id = self.reg_dict.get('reg_id')
- test_class = report.nodeid.split('::')[1]
- if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
- analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
- self.log.info('Analyzer Status is {}'.format(analyzer_status))
- else:
- self.log.info('Analyzer is not invoked as testcase failed in setup')
status = "unknown"
if testcase_name in self.testcase_dict:
status = self.testcase_dict[testcase_name]
@@ -1034,6 +1034,51 @@ def pytest_runtest_logreport(self, report):
else:
self.testcase_failtrace_dict[testcase_name] = None
+ # Add the testcase status to testcase_dict as error if the test failed in setup
+ try:
+ if report.when == 'setup' and report.outcome == 'failed':
+ testcase_name = self.get_test_name(report.nodeid)
+ self.testcase_dict[testcase_name] = 'error'
+ except Exception as e:
+ self.log.error("Error getting the testcase status for setup failure: {}".format(e))
+
+
+ @pytest.hookimpl(hookwrapper=True, trylast=True)
+ def pytest_runtest_makereport(self, item, call):
+ outcome = (yield)
+ if call.when =='call':
+ report = outcome.get_result()
+ testcase_name = self.get_test_name(report.nodeid)
+ if self.reg_dict:
+ reg_id = self.reg_dict.get('reg_id')
+ test_class = report.nodeid.split('::')[1]
+ analyzer_status = False
+ try:
+ if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
+ analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
+ self.log.info('Analyzer Status is {}'.format(analyzer_status))
+ else:
+ self.log.info('Analyzer is not invoked as testcase failed in setup')
+ if isinstance(analyzer_status, bool):
+ return
+ failures = json.loads(analyzer_status.get('failures',[]))
+ if len(failures):
+ self.log.error('Test case failed due to crash/traceback {}'.format(pformat(failures)))
+ test_outcome = 'failed'
+ report = TestReport(
+ report.nodeid,
+ report.location,
+ report.keywords,
+ test_outcome,
+ report.longrepr,
+ report.when,
+ report.sections,
+ report.duration,
+ )
+ outcome.force_result(report)
+ except:
+ self.log.error('Error while handling analyzer status')
+
def check_call_report(self, item, nextitem):
"""
@@ -1059,8 +1104,90 @@ def check_call_report(self, item, nextitem):
test_method.add_marker(pytest.mark.skipif("True"))
break
+ def request_retry(self, url, method, **kwargs):
+ """
+ Retry Connection to database.
+ Args:
+ url: String of URL of Cafy API.
+ method: String of 'GET', 'POST', 'PUT' or 'DELETE'.
+ **kwargs: Other Response arguments
+ :return:
+ """
+ HEADERS = {
+ 'content-type': "application/json",
+ 'authorization': "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXNzd29y"
+ "ZCI6Il9fd2VsbF9kb25lX2NhZnlfXyIsInVzZXJuYW1lIjoiX19jYWZ5"
+ "X3JvYm90X18ifQ.DQ1uVZeZG9mgmz7RMKaBebZcTIOVCvuzCYHjSrZHd"
+ "lI",
+ }
+
+ retries = Retry(total=5,
+ backoff_factor=1,
+ status_forcelist=[500, 502, 503, 504])
+ api_host = os.environ.get('CAFY_API_HOST')
+ url = "{}{}".format(api_host, url)
+ sessn = requests.Session()
+ sessn.mount('https://', HTTPAdapter(max_retries=retries))
+ sessn.mount('http://', HTTPAdapter(max_retries=retries))
+ kwargs['headers'] = HEADERS
+ data = kwargs.get('data')
+ kwargs.pop('data', "")
+ response = sessn.request(method=method, url=url, timeout=30, json=data, **kwargs)
+ if response.status_code == 200:
+ result = json.loads(s=response.text, object_pairs_hook=OrderedDict)
+ return result
+ else:
+ message = (f"An error occurred while trying to call API: {url}.\n\n"
+ f"Status code: {response.status_code}\n\n"
+ f"Content:\n{response.content}\n\n"
+ f"Text response:\n{response.text}\n\n")
+ self.log.info(message)
+ # pytest.exit(message)
+
+ def get_run_status(self):
+ """
+ This method is used to get the status of run.
+ :return:
+ """
+ run_id = os.environ.get('CAFY_RUN_ID')
+ url = f'/api/report/paused-run?run_id={run_id}'
+ method = 'GET'
+ return self.request_retry(url, method)
+
+ def set_run_status(self, method='PUT', **kwargs):
+ run_id = kwargs.get('run_id')
+ url = f'/api/update/run-state?run_id={run_id}'
+ return self.request_retry(url, method, **kwargs)
+
def pytest_runtest_protocol(self, item, nextitem):
# add to the hook
+ data = self.get_run_status().get('data')
+ waiting_interval = 30
+ current_time = datetime.utcnow()
+ start_time = datetime.utcnow()
+ if data.get('status') == 'P':
+ expiration_time = datetime.strptime(data.get('expiration_time'), '%Y-%m-%dT%H:%M:%S%z')
+ while current_time < expiration_time:
+ self.log.info(f'Sleeping for {waiting_interval} secs')
+ time.sleep(waiting_interval)
+ data = self.get_run_status().get('data')
+ expiration_time = datetime.strptime(data('expiration_time'), '%Y-%m-%dT%H:%M:%S%z')
+ if data['status'] == 'I':
+ self.log.info('Run is resumed, waited for {} !!'.format(str(datetime.now() - start_time)))
+ break
+ current_time = datetime.utcnow()
+ else:
+ method = 'PUT'
+ payload = {
+ 'status': 'I',
+ 'reason': 'Time limit exceeded',
+ 'run_id': os.environ.get('CAFY_RUN_ID')
+ }
+ self.set_run_status(method, data=payload)
+ msg = 'Waited for max time:{} to resume, so finally resuming now'.format(expiration_time - start_time)
+ self.log.info(msg)
+ # pytest.exit(msg)
+
item.ihook.pytest_runtest_logstart(
nodeid=item.nodeid, location=item.location,
)
@@ -1284,7 +1411,7 @@ def invoke_reg_on_failed_testcase(self, params, headers):
try:
url = "http://{0}:5001/startdebug/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to start collecting" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=1500)
if response.status_code == 200:
return response
else:
@@ -1301,7 +1428,7 @@ def invoke_rc_on_failed_testcase(self, params, headers):
try:
url = "http://{0}:5003/startrootcause/".format(CafyLog.debug_server)
self.log.info("Calling RC engine to start rootcause (url:%s)" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
return response
else:
@@ -1330,15 +1457,30 @@ def pytest_terminal_summary(self, terminalreporter):
if junitxml_file_path != _junitxml_filename:
copyfile(_junitxml_filename, junitxml_file_path)
os.chmod(junitxml_file_path, 0o775)
- if not self.no_email:
- self._sendemail()
- terminalreporter.write_line("\n TestCase Summary Status Table")
- temp_list = []
- for k,v in self.testcase_dict.items():
- temp_list.append((k,v))
- print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
-
+ @wrapt_timeout(600)
+ def terminal_summary_timeout(terminalreporter, *args):
+ try:
+ temp_list = []
+ terminalreporter.write_line("\n TestCase Summary Status Table")
+ for k,v in self.testcase_dict.items():
+ temp_list.append((k,v))
+ self.log.info("Printing the tabulated summary table")
+ print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
+ self.log.info("Preparing to send email")
+ if not self.no_email:
+ try:
+ self._sendemail()
+ except Exception as err:
+ self.log.error("Error when sending email: {err}".format(err=str(err)))
+ except TimeoutError as err:
+ trace = traceback.format_exc()
+ print(trace)
+ self.log.error("Encountered timeout of 600s in pytest_terminal_summary()")
+ raise err
+
+
+ terminal_summary_timeout(terminalreporter)
#Unset environ variables cafykit_mongo_learn & cafykit_mongo_read if set
if os.environ.get('cafykit_mongo_learn'):
del os.environ['cafykit_mongo_learn']
@@ -1403,7 +1545,7 @@ def _get_analyzer_log(self):
"debug_server_name": CafyLog.debug_server}
url = 'http://{0}:5001/get_analyzer_log/'.format(CafyLog.debug_server)
try:
- response = requests.get(url, data=params)
+ response = requests.get(url, data=params, timeout=300)
if response is not None and response.status_code == 200:
if response.text:
if 'Content-Disposition' in response.headers:
@@ -1435,15 +1577,22 @@ def pytest_sessionfinish(self):
url = 'http://{0}:5001/uploadcollectorlogfile/'.format(CafyLog.debug_server)
print("url = ", url)
self.log.info("Calling registration upload collector logfile service (url:%s)" %url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response is not None and response.status_code == 200:
if response.text:
- self.log.info ("Debug Collector logs: %s" %(response.text))
+ summary_log = response.text
+ if '+'*120 in response.text:
+ summary_log, verbose_log = response.text.split('+'*120)
+ self.log.info ("Debug Collector logs: %s" %(summary_log))
if 'Content-Disposition' in response.headers:
debug_collector_log_filename = response.headers['Content-Disposition'].split('filename=')[-1]
collector_log_file_full_path = os.path.join(CafyLog.work_dir,debug_collector_log_filename)
with open(collector_log_file_full_path, 'w') as f:
- f.write(response.text)
+ f.write(summary_log)
+ verbose_log_file_path = collector_log_file_full_path.replace("debug_collection.log",
+ "verbose_collection.log")
+ with open(verbose_log_file_path, 'w') as f:
+ f.write(verbose_log)
try:
DebugLibrary.convert_collector_logs_to_json(collector_log_file_full_path)
except:
@@ -1453,7 +1602,7 @@ def pytest_sessionfinish(self):
url = 'http://{0}:5001/deleteuploadedfiles/'.format(CafyLog.debug_server)
self.log.info("Calling registration delete upload file service (url:%s)" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
self.log.info("Topology and input files deleted from registration server")
else:
@@ -1537,7 +1686,11 @@ def __init__(self, terminalreporter, testcase_dict, testcase_failtrace_dict, arc
# Run Info
self.exec_host = platform.node()
self.python_version = platform.python_version()
- self. platform = platform.platform()
+ self.platform = platform.platform()
+ try:
+ self.cafykit_release = os.path.basename(os.environ.get("VIRTUAL_ENV"))
+ except:
+ self.cafykit_release = None
self.testbed = None
self.registration_id = CafyLog.registration_id
self.submitter = EmailReport.USER
@@ -1545,7 +1698,7 @@ def __init__(self, terminalreporter, testcase_dict, testcase_failtrace_dict, arc
self.topo_file = topo_file
self.run_dir = self.terminalreporter.startdir.strpath
try:
- self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master']).decode("utf-8").replace('\n', '')
+ self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master'], timeout=5).decode("utf-8").replace('\n', '')
except Exception:
self.git_commit_id = None
self.archive = CafyLog.work_dir
diff --git a/pytest_cafy/resources/mail_template.html b/pytest_cafy/resources/mail_template.html
index 65ba9f0..7d2bb62 100644
--- a/pytest_cafy/resources/mail_template.html
+++ b/pytest_cafy/resources/mail_template.html
@@ -1,38 +1,5 @@
-{% macro detailed_header(report) -%}
-
-Detailed Results
-
- 🕐 {{report.run_time}}
-
-
- View allure report >
-
-
-
- View Summary report >
-
-
-
-{%- endmacro %}
-
-
-{% macro detailed_data(report) -%}
-{% for name, status in report.testcase_dict.items() %}
-
- | {{name}} |
- {{status}} |
- {{report.testcase_failtrace_dict[name]}} |
-
-{% endfor %}
-{%- endmacro %}
-
-
{% macro summary(report) -%}
@@ -45,6 +12,15 @@ Detailed Results
{%- endmacro %}
+{% macro detailed_data(report) -%}
+{% for name, status in report.testcase_dict.items() %}
+
+ | {{name}} |
+ {{status}} |
+ {{report.testcase_failtrace_dict[name]}} |
+
+{% endfor %}
+{%- endmacro %}
@@ -57,6 +33,24 @@ Detailed Results
{{report.script_list}}
+
+ Summary
+
+
+
+ | Passed |
+ Xpassed |
+ Failed |
+ Skipped |
+ Xfailed |
+ Total |
+
+
+
+ {{ summary(report) }}
+
+
+
Run Info
@@ -67,10 +61,12 @@ Run Info
TestBed |
{{report.testbed}} |
+ {% if report.registration_id %}
| Registration ID |
{{report.registration_id}} |
+ {% endif %}
| Submitter |
{{report.submitter}} |
@@ -79,12 +75,18 @@ Run Info
Run directory |
{{report.run_dir}} |
-
+ {% if report.git_commit_id %}
| Git commit id |
{{report.git_commit_id}} |
-
+ {% endif %}
+ {% if report.cafykit_release %}
+
+ | Cafykit Release |
+ {{report.cafykit_release}} |
+
+ {% endif %}
| Script list |
{{report.script_list}} |
@@ -117,19 +119,10 @@ Run Info
Cafy repo |
{{report.cafy_repo}} |
-
| Exec host |
{{report.exec_host}} |
-
- | Python version |
- {{report.python_version}} |
-
-
- | Exec platform |
- {{report.platform}} |
-
| Topology file |
{{report.topo_file}} |
@@ -145,23 +138,6 @@ Run Info
-
- Summary
-
-
-
- | Passed |
- Xpassed |
- Failed |
- Skipped |
- Xfailed |
- Total |
-
-
-
- {{ summary(report) }}
-
-
Testcase Status Summary
@@ -178,43 +154,6 @@ Testcase Status Summary
-
-
- Build Info
-
- Image
- {{report.image}}
-
-
-
-
- | XR EFR |
- {{report.xr_efr}} |
-
-
- | XR workspace |
- {{report.xr_ws}} |
-
-
- | Calvados EFR |
- {{report.cal_efr}} |
-
-
- | Calvados workspace |
- {{report.cal_ws}} |
-
- {% if report.jenkins_url %}
-
- | Jenkins URL |
- {{report.jenkins_url}} |
-
- {% endif %}
-
-
-
-
-
- {{ detailed_header(report) }}