1#!/usr/bin/env python
2
3# Copyright (C) 2018 Bocoup LLC All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions
7# are met:
8#
9# 1. Redistributions of source code must retain the above
10# copyright notice, this list of conditions and the following
11# disclaimer.
12# 2. Redistributions in binary form must reproduce the above
13# copyright notice, this list of conditions and the following
14# disclaimer in the documentation and/or other materials
15# provided with the distribution.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
18# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
21# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
22# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
26# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
27# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28# SUCH DAMAGE.
29
30"""
31
32 This run-webkit-tests and analyzes the results then it attempts to update the
33 -expected.txt or the root TestExpecations file for failing test. This script is
34 intended to be uses after runnning Tools/Scripts/import-w3c-tests to assist in
35 creating a new test expectation baseline after importing new tests from
36 web-platform-tests.
37
38 The script will update the expectations files according to the following rules:
39
40 Initially the script runs Tools/Script/run-webkit-tests on the specified tests
41 or directories to generate a baseline.
42
43 - Missing tests will be re-run to ensure they are not flaky.
44
45 - Crashing or Timing out tests will be added to the root TestExpecations
46 files with [ Skip ] directive.
47
48 - Tests that pass and are expected to fail will remove the failing
49 directive from the TestExpecations file and will be run again.
50
51 - Failing ref tests will be added to the root TestExpecations files with
52 [ ImageOnlyFailure ] directive.
53
54 - Failing testharness tests will be run again with the --reset-results flag
55 to reset the -expected.txt file. If testharness tests fail multiple times
56 they will be added to the root TestExpecations files with [ Failure ]
57 directive.
58
59 - Flaky tests will be added to the root TestExpecations files with all of
60 their failure state directives.
61"""
62
63import argparse
64import json
65from subprocess import Popen
66import io
67import os
68from sets import Set
69import logging
70
71from webkitpy.layout_tests.run_webkit_tests import parse_args
72from webkitpy.common.host import Host
73
74_log = logging.getLogger(__name__)
75
76
77def configure_logging():
78 class LogHandler(logging.StreamHandler):
79
80 def format(self, record):
81 if record.levelno > logging.INFO:
82 return "%s: %s" % (record.levelname, record.getMessage())
83 return record.getMessage()
84
85 logger = logging.getLogger()
86 logger.setLevel(logging.INFO)
87 handler = LogHandler()
88 handler.setLevel(logging.INFO)
89 logger.addHandler(handler)
90 return handler
91
92
93# TODO
94# Add documentation of argorithm for updating test expectations
95# cleanup code to follow webkitpy standards
96# add unittests
97
98def main(_argv, _stdout, _stderr):
99 configure_logging()
100
101 test_updater = TestExpecationUpdater(Host(), _argv)
102 test_updater.do_update()
103
104
105class TestExpecationUpdater(object):
106 def __init__(self, host, args):
107 self._host = host
108 options, path_args = parse_args(args)
109 self._options = options
110 option_args = list(Set(args).difference(Set(path_args)))
111 # preserve original order of arguments
112 option_args = [arg for arg in args if arg in option_args]
113
114 self._option_args = option_args
115 self._base_test = path_args
116 self._port = host.port_factory.get(options.platform, options)
117
118 def do_update(self):
119 # Run tests once to get a baseline
120 self._run_webkit_tests(self._base_test)
121 with open(self._results_file()) as f:
122 data = json.load(f)
123 tests = self._pre_process_tests(data['tests'])
124
125 for test in tests:
126 _log.info('%s/%s Processing test %s' % (tests.index(test), len(tests), test['name']))
127 self._update_expectation_for_failing_test(test)
128
129 def _update_expectation_for_failing_test(self, test, post_reset_result=False, previous_result=None):
130 _log.debug(test)
131 if test.get('report') == 'FLAKY' or previous_result:
132 return self._flaky_test(test, previous_result)
133 if test.get('report') == 'MISSING':
134 return self._missing_test(test)
135 if test.get('report') == 'REGRESSION' and test.get('expected') == 'CRASH':
136 self._unexpected_pass_test(test)
137 if test.get('actual') == 'CRASH':
138 return self._crash_test(test)
139 if test.get('actual') == 'TIMEOUT':
140 return self._timeout_test(test)
141 if test.get('actual') == 'PASS':
142 return self._unexpected_pass_test(test)
143 if test.get('actual') == 'IMAGE':
144 return self._failing_ref_test(test)
145 if test.get('actual') == 'TEXT MISSING':
146 return self._missing_test(test)
147 if test.get('actual') == 'TEXT' and post_reset_result:
148 return self._failing_testharness_test(test)
149 if test.get('actual') == 'TEXT IMAGE+TEXT':
150 return self._failing_testharness_test(test)
151 if test.get('actual') == 'TEXT':
152 return self._reset_testharness_test(test)
153 raise NotImplementedError('The test updater decision engine could not figure out how to handle test: %s' % json.dumps(test))
154
155 def _flaky_test(self, test, previous_result=None):
156 expectations = test['actual'].split(' ')
157 if previous_result:
158 expectations = expectations + previous_result['actual'].split(' ')
159 expectations = list(Set(expectations + ['FAIL']))
160 self._update_test_expectation(test['name'], self._render_expectations(expectations))
161 self._run_webkit_tests([test['name']])
162 result = self._extract_failing_test_result(test)
163 if result:
164 self._update_expectation_for_failing_test(result, previous_result=test)
165
166 def _missing_test(self, test):
167 self._run_webkit_tests([test['name']])
168 result = self._extract_failing_test_result(test)
169 if result:
170 # Test is still failing, attempt to re-classify
171 self._update_expectation_for_failing_test(result)
172
173 def _unexpected_pass_test(self, test):
174 self._remove_test_expectation(test['name'])
175
176 self._run_webkit_tests([test['name']])
177 result = self._extract_failing_test_result(test)
178 if result:
179 self._update_expectation_for_failing_test(result, previous_result=test)
180
181 def _crash_test(self, test):
182 self._update_test_expectation(test['name'], 'Skip')
183
184 def _timeout_test(self, test):
185 self._update_test_expectation(test['name'], 'Skip')
186
187 def _failing_ref_test(self, test):
188 self._update_test_expectation(test['name'], 'ImageOnlyFailure')
189 self._run_webkit_tests([test['name']])
190 result = self._extract_failing_test_result(test)
191 if result:
192 # Test is still failing, attempt to re-classify
193 self._update_expectation_for_failing_test(result, previous_result=test)
194
195 def _reset_testharness_test(self, test):
196 self._run_webkit_tests([test['name']], reset_results=True)
197 self._run_webkit_tests([test['name']])
198 result = self._extract_failing_test_result(test)
199 if result:
200 self._update_expectation_for_failing_test(result, post_reset_result=True)
201
202 def _failing_testharness_test(self, test):
203 self._update_test_expectation(test['name'], 'Failure')
204 self._run_webkit_tests([test['name']])
205 result = self._extract_failing_test_result(test)
206 if result:
207 self._update_expectation_for_failing_test(result, previous_result=test)
208
209 def _run_webkit_tests(self, test_files, reset_results=False):
210 args = ['Tools/Scripts/run-webkit-tests'] + self._option_args
211 if reset_results:
212 args.append('--reset-results')
213
214 args = args + test_files
215
216 _log.info('Running webkit tests for: %s' % test_files)
217 p = Popen(args)
218 return p.wait()
219
220 def _test_expectations_path(self):
221 return self._port.path_to_generic_test_expectations_file()
222
223 def _results_file(self):
224 options = self._options
225 return self._host.filesystem.join(options.build_directory, options.configuration, 'layout-test-results/full-results.json')
226
227 def _render_expectations(self, failures):
228 expectation_map = {
229 'TIMEOUT': 'Timeout',
230 'FAIL': 'Failure',
231 'TEXT': 'Failure',
232 'IMAGE+TEXT': 'Failure',
233 'MISSING': '',
234 'PASS': 'Pass',
235 'IMAGE': 'ImageOnlyFailure',
236 }
237 return ' '.join([expectation_map[failure] for failure in failures])
238
239 def _update_test_expectation(self, test, expectation):
240 self._remove_test_expectation(test)
241 with open(self._test_expectations_path(), 'a') as myfile:
242 _log.info('Updating TestExpectations %s [ %s ]' % (test, expectation))
243 myfile.write('\n%s [ %s ]\n' % (test, expectation))
244
245 def _remove_test_expectation(self, test_name):
246 for path in self._port.expectations_files():
247 if os.path.isfile(path):
248 self._remove_test_expectation_from_path(path, test_name)
249
250 def _remove_test_expectation_from_path(self, expectation_file, test_name):
251 with io.open(expectation_file, 'r', encoding="utf-8") as fd:
252 lines = fd.readlines()
253
254 with io.open(expectation_file, 'w', encoding="utf-8") as fd:
255 for line in lines:
256 if test_name not in line:
257 fd.write(line)
258
259 def _extract_failing_test_result(self, test):
260 with open(self._results_file()) as f:
261 data = json.load(f)
262 tests = self._pre_process_tests(data['tests'])
263 matching_tests = [t for t in tests if t['name'] == test['name']]
264
265 if len(matching_tests) and not self._results_match_expectation(matching_tests[0]):
266 return matching_tests[0]
267 else:
268 return None
269
270 def _pre_process_tests(self, test_dict):
271 tests = self._flatten_path(test_dict)
272 processed_tests = []
273 for file_name, results in tests.items():
274 results['name'] = file_name
275 processed_tests.append(results)
276
277 processed_tests = [test for test in processed_tests if not self._results_match_expectation(test)]
278
279 return processed_tests
280
281 def _results_match_expectation(self, result):
282 if 'FAIL' in result['expected'] and result['actual'] == 'TEXT':
283 return True
284 if result['actual'] in result['expected']:
285 return True
286 return False
287
288 def _flatten_path(self, obj):
289 to_return = {}
290 for k, v in obj.items():
291 if 'expected' in v:
292 # terminary node
293 to_return[k] = v
294 pass
295 else:
296 flat_object = self._flatten_path(v)
297 for k2, v2 in flat_object.items():
298 to_return[k + '/' + k2] = v2
299 return to_return