294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
1 |
# IVLE - Informatics Virtual Learning Environment
|
2 |
# Copyright (C) 2007-2008 The University of Melbourne
|
|
3 |
#
|
|
4 |
# This program is free software; you can redistribute it and/or modify
|
|
5 |
# it under the terms of the GNU General Public License as published by
|
|
6 |
# the Free Software Foundation; either version 2 of the License, or
|
|
7 |
# (at your option) any later version.
|
|
8 |
#
|
|
9 |
# This program is distributed in the hope that it will be useful,
|
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
12 |
# GNU General Public License for more details.
|
|
13 |
#
|
|
14 |
# You should have received a copy of the GNU General Public License
|
|
15 |
# along with this program; if not, write to the Free Software
|
|
16 |
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
17 |
||
18 |
# Module: TestFramework
|
|
19 |
# Author: Dilshan Angampitiya
|
|
20 |
# Date: 24/1/2008
|
|
21 |
||
22 |
# Brief description of the Module# define custom exceptions
|
|
23 |
# use exceptions for all errors found in testing
|
|
24 |
||
25 |
import sys, StringIO, copy |
|
26 |
||
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
27 |
# student error or author error
|
28 |
# errors in student code get handled internally
|
|
29 |
# errors in solution code get passed up
|
|
30 |
class ScriptExecutionError(Exception): |
|
31 |
"""Runtime error in the student code or solution code"""
|
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
32 |
def __init__(self, exc_info): |
33 |
cla, exc, trbk = exc_info |
|
34 |
self._name = cla.__name__ |
|
35 |
self._detail = str(exc) |
|
310
by dilshan_a
Add lineno to exception info. |
36 |
self._trbk = trbk |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
37 |
|
38 |
def is_critical(self): |
|
39 |
if ( self._name == 'FunctionNotFoundError' |
|
40 |
or self._name == 'SyntaxError' |
|
41 |
or self._name == 'IndentationError'): |
|
42 |
return True |
|
43 |
else: |
|
44 |
return False |
|
45 |
||
46 |
def to_dict(self): |
|
310
by dilshan_a
Add lineno to exception info. |
47 |
import traceback |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
48 |
return {'name': self._name, |
49 |
'detail': self._detail, |
|
311
by dilshan_a
Last commit was buggy. |
50 |
'critical': self.is_critical(), |
310
by dilshan_a
Add lineno to exception info. |
51 |
'lineno': traceback.tb_lineno(self._trbk) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
52 |
}
|
53 |
||
54 |
def __str__(self): |
|
55 |
return self._name + " - " + str(self._detail) |
|
56 |
||
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
57 |
# author error
|
58 |
class TestCreationError(Exception): |
|
59 |
"""An error occured while creating the test suite or one of its components"""
|
|
60 |
def __init__(self, reason): |
|
61 |
self._reason = reason |
|
62 |
||
63 |
def __str__(self): |
|
64 |
return self._reason |
|
65 |
||
66 |
# author error
|
|
67 |
class TestError(Exception): |
|
68 |
"""Runtime error in the testing framework outside of the provided or student code"""
|
|
69 |
def __init__(self, exc_info): |
|
70 |
cla, exc, trbk = exc_info |
|
71 |
self._name = cla.__name__ |
|
72 |
self._detail = str(exc) |
|
73 |
self._exc_info = exc_info |
|
74 |
||
75 |
def exc_info(self): |
|
76 |
return self._exc_info |
|
77 |
||
78 |
def __str__(self): |
|
79 |
return "Error testing solution against attempt: %s - %s" %(self._name, self._detail) |
|
80 |
||
81 |
# author error
|
|
82 |
# raised when expected file not found in solution output
|
|
83 |
# Always gets caught and passed up as a TestError
|
|
84 |
class FileNotFoundError(Exception): |
|
85 |
def __init__(self, filename): |
|
86 |
self._filename = filename |
|
87 |
||
88 |
def __str__(self): |
|
89 |
return "File %s not found in output" %(self._filename) |
|
90 |
||
91 |
||
92 |
# Error encountered when executing solution or attempt code
|
|
93 |
# Always gets caught and passed up in a ScriptExecutionError
|
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
94 |
class FunctionNotFoundError(Exception): |
95 |
"""This error is returned when a function was expected in a
|
|
96 |
test case but was not found"""
|
|
97 |
def __init__(self, function_name): |
|
98 |
self.function_name = function_name |
|
99 |
||
100 |
def __str__(self): |
|
101 |
return "Function " + self.function_name + " not found" |
|
102 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
103 |
class TestCasePart: |
104 |
"""
|
|
105 |
A part of a test case which compares a subset of the input files or file streams.
|
|
106 |
This can be done either with a comparision function, or by comparing directly, after
|
|
107 |
applying normalisations.
|
|
108 |
"""
|
|
109 |
# how to make this work? atm they seem to get passed the class as a first arg
|
|
110 |
ident =lambda x: x |
|
111 |
ignore = lambda x: None |
|
112 |
match = lambda x,y: x==y |
|
113 |
always_match = lambda x,y: True |
|
114 |
true = lambda *x: True |
|
115 |
false = lambda *x: False |
|
116 |
||
117 |
def __init__(self, desc, default='match'): |
|
118 |
"""Initialise with a description and a default behavior for output
|
|
119 |
If default is match, unspecified files are matched exactly
|
|
120 |
If default is ignore, unspecified files are ignored
|
|
121 |
The default default is match.
|
|
122 |
"""
|
|
123 |
self._desc = desc |
|
124 |
self._default = default |
|
125 |
if default == 'ignore': |
|
126 |
self._default_func = lambda *x: True |
|
127 |
else: |
|
128 |
self._default_func = lambda x,y: x==y |
|
129 |
||
130 |
self._file_tests = {} |
|
131 |
self._stdout_test = ('check', self._default_func) |
|
132 |
self._stderr_test = ('check', self._default_func) |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
133 |
self._exception_test = ('check', self._default_func) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
134 |
self._result_test = ('check', self._default_func) |
135 |
||
136 |
def get_description(self): |
|
137 |
"Getter for description"
|
|
138 |
return self._desc |
|
139 |
||
140 |
def _set_default_function(self, function, test_type): |
|
141 |
""""Ensure test type is valid and set function to a default
|
|
142 |
if not specified"""
|
|
143 |
||
144 |
if test_type not in ['norm', 'check']: |
|
145 |
raise TestCreationError("Invalid test type in %s" %self._desc) |
|
146 |
||
147 |
if function == '': |
|
148 |
if test_type == 'norm': function = lambda x: x |
|
149 |
else: function = lambda x,y: x==y |
|
150 |
||
151 |
return function |
|
152 |
||
153 |
def _validate_function(self, function, included_code): |
|
154 |
"""Create a function object from the given string.
|
|
155 |
If a valid function object cannot be created, raise and error.
|
|
156 |
"""
|
|
157 |
if not callable(function): |
|
158 |
try: |
|
159 |
exec "__f__ = %s" %function in included_code |
|
160 |
except: |
|
161 |
raise TestCreationError("Invalid function %s" %function) |
|
162 |
||
163 |
f = included_code['__f__'] |
|
164 |
||
165 |
if not callable(f): |
|
166 |
raise TestCreationError("Invalid function %s" %function) |
|
167 |
else: |
|
168 |
f = function |
|
169 |
||
170 |
return f |
|
171 |
||
172 |
def validate_functions(self, included_code): |
|
173 |
"""Ensure all functions used by the test cases exist and are callable.
|
|
174 |
Also covert their string representations to function objects.
|
|
175 |
This can only be done once all the include code has been specified.
|
|
176 |
"""
|
|
177 |
(test_type, function) = self._stdout_test |
|
178 |
self._stdout_test = (test_type, self._validate_function(function, included_code)) |
|
179 |
||
180 |
(test_type, function) = self._stderr_test |
|
181 |
self._stderr_test = (test_type, self._validate_function(function, included_code)) |
|
182 |
||
183 |
for filename, (test_type, function) in self._file_tests.items(): |
|
184 |
self._file_tests[filename] = (test_type, self._validate_function(function, included_code)) |
|
185 |
||
186 |
def add_result_test(self, function, test_type='norm'): |
|
187 |
"Test part that compares function return values"
|
|
188 |
function = self._set_default_function(function, test_type) |
|
189 |
self._result_test = (test_type, function) |
|
190 |
||
191 |
||
192 |
def add_stdout_test(self, function, test_type='norm'): |
|
193 |
"Test part that compares stdout"
|
|
194 |
function = self._set_default_function(function, test_type) |
|
195 |
self._stdout_test = (test_type, function) |
|
196 |
||
197 |
||
198 |
def add_stderr_test(self, function, test_type='norm'): |
|
199 |
"Test part that compares stderr"
|
|
200 |
function = self._set_default_function(function, test_type) |
|
201 |
self._stderr_test = (test_type, function) |
|
202 |
||
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
203 |
def add_exception_test(self, function, test_type='norm'): |
204 |
"Test part that compares stderr"
|
|
205 |
function = self._set_default_function(function, test_type) |
|
206 |
self._exception_test = (test_type, function) |
|
207 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
208 |
def add_file_test(self, filename, function, test_type='norm'): |
209 |
"Test part that compares the contents of a specified file"
|
|
210 |
function = self._set_default_function(function, test_type) |
|
211 |
self._file_tests[filename] = (test_type, function) |
|
212 |
||
213 |
def _check_output(self, solution_output, attempt_output, test_type, f): |
|
214 |
"""Compare solution output and attempt output using the
|
|
215 |
specified comparision function.
|
|
216 |
"""
|
|
217 |
# converts unicode to string
|
|
218 |
if type(solution_output) == unicode: |
|
219 |
solution_output = str(solution_output) |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
220 |
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
221 |
if type(attempt_output) == unicode: |
222 |
attempt_output = str(attempt_output) |
|
223 |
||
224 |
if test_type == 'norm': |
|
225 |
return f(solution_output) == f(attempt_output) |
|
226 |
else: |
|
227 |
return f(solution_output, attempt_output) |
|
228 |
||
229 |
def run(self, solution_data, attempt_data): |
|
230 |
"""Run the tests to compare the solution and attempt data
|
|
231 |
Returns the empty string is the test passes, or else an error message.
|
|
232 |
"""
|
|
233 |
||
234 |
# check function return value (None for scripts)
|
|
235 |
(test_type, f) = self._result_test |
|
236 |
if not self._check_output(solution_data['result'], attempt_data['result'], test_type, f): |
|
237 |
return 'function return value does not match' |
|
238 |
||
239 |
# check stdout
|
|
240 |
(test_type, f) = self._stdout_test |
|
241 |
if not self._check_output(solution_data['stdout'], attempt_data['stdout'], test_type, f): |
|
242 |
return 'stdout does not match' |
|
243 |
||
244 |
#check stderr
|
|
245 |
(test_type, f) = self._stderr_test |
|
246 |
if not self._check_output(solution_data['stderr'], attempt_data['stderr'], test_type, f): |
|
247 |
return 'stderr does not match' |
|
248 |
||
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
249 |
#check exception
|
250 |
(test_type, f) = self._exception_test |
|
251 |
if not self._check_output(solution_data['exception'], attempt_data['exception'], test_type, f): |
|
252 |
return 'exception does not match' |
|
253 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
254 |
|
255 |
solution_files = solution_data['modified_files'] |
|
256 |
attempt_files = attempt_data['modified_files'] |
|
257 |
||
258 |
# check files indicated by test
|
|
259 |
for (filename, (test_type, f)) in self._file_tests.items(): |
|
260 |
if filename not in solution_files: |
|
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
261 |
raise FileNotFoundError(filename) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
262 |
elif filename not in attempt_files: |
263 |
return filename + ' not found' |
|
264 |
elif not self._check_output(solution_files[filename], attempt_files[filename], test_type, f): |
|
265 |
return filename + ' does not match' |
|
266 |
||
267 |
if self._default == 'ignore': |
|
268 |
return '' |
|
269 |
||
270 |
# check files found in solution, but not indicated by test
|
|
271 |
for filename in [f for f in solution_files if f not in self._file_tests]: |
|
272 |
if filename not in attempt_files: |
|
273 |
return filename + ' not found' |
|
274 |
elif not self._check_output(solution_files[filename], attempt_files[filename], 'match', lambda x,y: x==y): |
|
275 |
return filename + ' does not match' |
|
276 |
||
277 |
# check if attempt has any extra files
|
|
278 |
for filename in [f for f in attempt_files if f not in solution_files]: |
|
279 |
return "Unexpected file found: " + filename |
|
280 |
||
281 |
# Everything passed with no problems
|
|
282 |
return '' |
|
283 |
||
284 |
class TestCase: |
|
285 |
"""
|
|
286 |
A set of tests with a common inputs
|
|
287 |
"""
|
|
288 |
def __init__(self, name='', function=None, stdin='', filespace=None, global_space=None): |
|
289 |
"""Initialise with name and optionally, a function to test (instead of the entire script)
|
|
290 |
The inputs stdin, the filespace and global variables can also be specified at
|
|
291 |
initialisation, but may also be set later.
|
|
292 |
"""
|
|
293 |
if global_space == None: |
|
294 |
global_space = {} |
|
295 |
if filespace == None: |
|
296 |
filespace = {} |
|
297 |
||
298 |
self._name = name |
|
299 |
||
300 |
if function == '': function = None |
|
301 |
self._function = function |
|
302 |
self._list_args = [] |
|
303 |
self._keyword_args = {} |
|
304 |
||
305 |
# stdin must have a newline at the end for raw_input to work properly
|
|
306 |
if stdin[-1:] != '\n': stdin += '\n' |
|
307 |
||
308 |
self._stdin = stdin |
|
309 |
self._filespace = TestFilespace(filespace) |
|
310 |
self._global_space = global_space |
|
311 |
self._parts = [] |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
312 |
self._allowed_exceptions = set() |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
313 |
|
314 |
def set_stdin(self, stdin): |
|
315 |
""" Set the given string as the stdin for this test case"""
|
|
316 |
self._stdin = stdin |
|
317 |
||
318 |
def add_file(self, filename, data): |
|
319 |
""" Insert the given filename-data pair into the filespace for this test case"""
|
|
320 |
self._filespace.add_file(filename, data) |
|
321 |
||
322 |
def add_variable(self, variable, value): |
|
323 |
""" Add the given varibale-value pair to the initial global environment
|
|
324 |
for this test case.
|
|
325 |
Throw and exception if thevalue cannot be paresed.
|
|
326 |
"""
|
|
327 |
||
328 |
try: |
|
329 |
self._global_space[variable] = eval(value) |
|
330 |
except: |
|
331 |
raise TestCreationError("Invalid value for variable %s: %s" %(variable, value)) |
|
332 |
||
333 |
def add_arg(self, value, name=None): |
|
334 |
""" Add a value to the argument list. This only applies when testing functions.
|
|
335 |
By default arguments are not named, but if they are, they become keyword arguments.
|
|
336 |
"""
|
|
337 |
try: |
|
338 |
if name == None or name == '': |
|
339 |
self._list_args.append(eval(value)) |
|
340 |
else: |
|
341 |
self._keyword_args[name] = value |
|
342 |
except: |
|
343 |
raise TestCreationError("Invalid value for function argument: %s" %value) |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
344 |
|
345 |
def add_exception(self, exception_name): |
|
346 |
self._allowed_exceptions.add(exception_name) |
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
347 |
|
348 |
def add_part(self, test_part): |
|
349 |
""" Add a TestPart to this test case"""
|
|
350 |
self._parts.append(test_part) |
|
351 |
||
352 |
def validate_functions(self, included_code): |
|
353 |
""" Validate all the functions in each part in this test case
|
|
354 |
This can only be done once all the include code has been specified.
|
|
355 |
"""
|
|
356 |
for part in self._parts: |
|
357 |
part.validate_functions(included_code) |
|
358 |
||
359 |
def get_name(self): |
|
360 |
""" Get the name of the test case """
|
|
361 |
return self._name |
|
362 |
||
302
by dilshan_a
Updated test framework so that student's code is passed in as a string. |
363 |
def run(self, solution, attempt_code): |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
364 |
""" Run the solution and the attempt with the inputs specified for this test case.
|
365 |
Then pass the outputs to each test part and collate the results.
|
|
366 |
"""
|
|
367 |
case_dict = {} |
|
368 |
case_dict['name'] = self._name |
|
369 |
||
370 |
# Run solution
|
|
371 |
try: |
|
372 |
global_space_copy = copy.deepcopy(self._global_space) |
|
373 |
solution_data = self._execstring(solution, global_space_copy) |
|
374 |
||
375 |
# if we are just testing a function
|
|
376 |
if not self._function == None: |
|
377 |
if self._function not in global_space_copy: |
|
378 |
raise FunctionNotFoundError(self._function) |
|
379 |
solution_data = self._run_function(lambda: global_space_copy[self._function](*self._list_args, **self._keyword_args)) |
|
380 |
||
381 |
except: |
|
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
382 |
raise ScriptExecutionError(sys.exc_info()) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
383 |
|
384 |
# Run student attempt
|
|
385 |
try: |
|
386 |
global_space_copy = copy.deepcopy(self._global_space) |
|
302
by dilshan_a
Updated test framework so that student's code is passed in as a string. |
387 |
attempt_data = self._execstring(attempt_code, global_space_copy) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
388 |
|
389 |
# if we are just testing a function
|
|
390 |
if not self._function == None: |
|
391 |
if self._function not in global_space_copy: |
|
392 |
raise FunctionNotFoundError(self._function) |
|
393 |
attempt_data = self._run_function(lambda: global_space_copy[self._function](*self._list_args, **self._keyword_args)) |
|
394 |
except: |
|
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
395 |
case_dict['exception'] = ScriptExecutionError(sys.exc_info()).to_dict() |
304
by dilshan_a
Added documentation of output of TestSuite. |
396 |
case_dict['passed'] = False |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
397 |
return case_dict |
398 |
||
399 |
results = [] |
|
304
by dilshan_a
Added documentation of output of TestSuite. |
400 |
passed = True |
401 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
402 |
# generate results
|
403 |
for test_part in self._parts: |
|
306
by dilshan_a
Consolidated SolutionError and AttemptError into ScriptExecutionError. |
404 |
try: |
405 |
result = test_part.run(solution_data, attempt_data) |
|
406 |
except: |
|
407 |
raise TestError(sys.exc_info()) |
|
408 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
409 |
result_dict = {} |
410 |
result_dict['description'] = test_part.get_description() |
|
411 |
result_dict['passed'] = (result == '') |
|
412 |
if result_dict['passed'] == False: |
|
413 |
result_dict['error_message'] = result |
|
304
by dilshan_a
Added documentation of output of TestSuite. |
414 |
passed = False |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
415 |
|
416 |
results.append(result_dict) |
|
417 |
||
418 |
case_dict['parts'] = results |
|
304
by dilshan_a
Added documentation of output of TestSuite. |
419 |
case_dict['passed'] = passed |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
420 |
|
421 |
return case_dict |
|
422 |
||
423 |
def _execfile(self, filename, global_space): |
|
424 |
""" Execute the file given by 'filename' in global_space, and return the outputs. """
|
|
425 |
self._initialise_global_space(global_space) |
|
426 |
data = self._run_function(lambda: execfile(filename, global_space)) |
|
427 |
return data |
|
428 |
||
429 |
def _execstring(self, string, global_space): |
|
430 |
""" Execute the given string in global_space, and return the outputs. """
|
|
431 |
self._initialise_global_space(global_space) |
|
305
by dilshan_a
Neated up execution of strings in TestCase. |
432 |
|
433 |
def f(): |
|
434 |
exec string in global_space |
|
435 |
||
436 |
data = self._run_function(f) |
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
437 |
return data |
438 |
||
439 |
def _initialise_global_space(self, global_space): |
|
440 |
""" Modify the provided global_space so that file, open and raw_input are redefined
|
|
441 |
to use our methods instead.
|
|
442 |
"""
|
|
443 |
self._current_filespace_copy = self._filespace.copy() |
|
444 |
global_space['file'] = lambda filename, mode='r', bufsize=-1: self._current_filespace_copy.openfile(filename, mode) |
|
445 |
global_space['open'] = global_space['file'] |
|
446 |
global_space['raw_input'] = lambda x=None: raw_input() |
|
447 |
return global_space |
|
448 |
||
449 |
def _run_function(self, function): |
|
450 |
""" Run the provided function with the provided stdin, capturing stdout and stderr
|
|
451 |
and the return value.
|
|
452 |
Return all the output data.
|
|
453 |
"""
|
|
454 |
import sys, StringIO |
|
455 |
sys_stdout, sys_stdin, sys_stderr = sys.stdout, sys.stdin, sys.stderr |
|
456 |
||
457 |
output_stream, input_stream, error_stream = StringIO.StringIO(), StringIO.StringIO(self._stdin), StringIO.StringIO() |
|
458 |
sys.stdout, sys.stdin, sys.stderr = output_stream, input_stream, error_stream |
|
459 |
||
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
460 |
result = None |
461 |
exception_name = None |
|
462 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
463 |
try: |
305
by dilshan_a
Neated up execution of strings in TestCase. |
464 |
result = function() |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
465 |
except: |
466 |
sys.stdout, sys.stdin, sys.stderr = sys_stdout, sys_stdin, sys_stderr |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
467 |
exception_name = sys.exc_info()[0].__name__ |
468 |
if exception_name not in self._allowed_exceptions: |
|
469 |
raise
|
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
470 |
|
471 |
sys.stdout, sys.stdin, sys.stderr = sys_stdout, sys_stdin, sys_stderr |
|
472 |
||
473 |
self._current_filespace_copy.flush_all() |
|
474 |
||
475 |
return {'result': result, |
|
299
by dilshan_a
Test framework now handles exceptions as valid outputs for scripts. |
476 |
'exception': exception_name, |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
477 |
'stdout': output_stream.getvalue(), |
478 |
'stderr': output_stream.getvalue(), |
|
479 |
'modified_files': self._current_filespace_copy.get_modified_files()} |
|
480 |
||
481 |
class TestSuite: |
|
482 |
"""
|
|
483 |
The complete collection of test cases for a given problem
|
|
484 |
"""
|
|
485 |
def __init__(self, name, solution=None): |
|
486 |
"""Initialise with the name of the test suite (the problem name) and the solution.
|
|
487 |
The solution may be specified later.
|
|
488 |
"""
|
|
489 |
self._solution = solution |
|
490 |
self._name = name |
|
491 |
self._tests = [] |
|
492 |
self.add_include_code("") |
|
493 |
||
494 |
def add_solution(self, solution): |
|
495 |
" Specifiy the solution script for this problem "
|
|
496 |
self._solution = solution |
|
497 |
||
498 |
def has_solution(self): |
|
499 |
" Returns true if a soltion has been provided "
|
|
500 |
return self._solution != None |
|
501 |
||
502 |
def add_include_code(self, include_code = ''): |
|
503 |
""" Add include code that may be used by the test cases during
|
|
504 |
comparison of outputs.
|
|
505 |
"""
|
|
506 |
||
507 |
# if empty, make sure it can still be executed
|
|
508 |
if include_code == "": |
|
509 |
include_code = "pass" |
|
510 |
self._include_code = str(include_code) |
|
511 |
||
512 |
include_space = {} |
|
513 |
try: |
|
514 |
exec self._include_code in include_space |
|
515 |
except: |
|
516 |
raise TestCreationError("Bad include code") |
|
517 |
||
518 |
self._include_space = include_space |
|
519 |
||
520 |
def add_case(self, test_case): |
|
521 |
""" Add a TestCase, then validate all functions inside test case
|
|
522 |
now that the include code is known
|
|
523 |
"""
|
|
524 |
self._tests.append(test_case) |
|
525 |
test_case.validate_functions(self._include_space) |
|
526 |
||
304
by dilshan_a
Added documentation of output of TestSuite. |
527 |
def run_tests(self, attempt_code, stop_on_fail=True): |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
528 |
" Run all test cases and collate the results "
|
529 |
||
530 |
problem_dict = {} |
|
531 |
problem_dict['name'] = self._name |
|
532 |
||
533 |
test_case_results = [] |
|
309
by dilshan_a
Added a passed key to return value of problem suite. |
534 |
passed = True |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
535 |
for test in self._tests: |
302
by dilshan_a
Updated test framework so that student's code is passed in as a string. |
536 |
result_dict = test.run(self._solution, attempt_code) |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
537 |
if 'exception' in result_dict and result_dict['exception']['critical']: |
538 |
# critical error occured, running more cases is useless
|
|
539 |
# FunctionNotFound, Syntax, Indentation
|
|
540 |
problem_dict['critical_error'] = result_dict['exception'] |
|
309
by dilshan_a
Added a passed key to return value of problem suite. |
541 |
problem_dict['passed'] = False |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
542 |
return problem_dict |
543 |
||
544 |
test_case_results.append(result_dict) |
|
304
by dilshan_a
Added documentation of output of TestSuite. |
545 |
|
309
by dilshan_a
Added a passed key to return value of problem suite. |
546 |
if not result_dict['passed']: |
547 |
passed = False |
|
548 |
if stop_on_fail: |
|
549 |
break
|
|
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
550 |
|
551 |
problem_dict['cases'] = test_case_results |
|
309
by dilshan_a
Added a passed key to return value of problem suite. |
552 |
problem_dict['passed'] = passed |
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
553 |
return problem_dict |
554 |
||
555 |
def get_name(self): |
|
556 |
return self._name |
|
557 |
||
558 |
class TestFilespace: |
|
559 |
"""
|
|
560 |
Our dummy file system which is accessed by code being tested.
|
|
561 |
Implemented as a dictionary which maps filenames to strings
|
|
562 |
"""
|
|
563 |
def __init__(self, files=None): |
|
564 |
"Initialise, optionally with filename-filedata pairs"
|
|
565 |
||
566 |
if files == None: |
|
567 |
files = {} |
|
568 |
||
569 |
# dict mapping files to strings
|
|
570 |
self._files = {} |
|
571 |
self._files.update(files) |
|
572 |
# set of file names
|
|
573 |
self._modified_files = set([]) |
|
574 |
# dict mapping files to stringIO objects
|
|
575 |
self._open_files = {} |
|
576 |
||
577 |
def add_file(self, filename, data): |
|
578 |
" Add a file to the filespace "
|
|
579 |
self._files[filename] = data |
|
580 |
||
581 |
def openfile(self, filename, mode='r'): |
|
582 |
""" Open a file from the filespace with the given mode.
|
|
583 |
Return a StringIO subclass object with the file contents.
|
|
584 |
"""
|
|
309
by dilshan_a
Added a passed key to return value of problem suite. |
585 |
# currently very messy, needs to be cleaned up
|
586 |
# Probably most of this should be in the initialiser to the TestStringIO
|
|
587 |
||
294
by mattgiuca
Added application: tutorialservice. Will be used as the Ajax backend for |
588 |
import re |
589 |
||
590 |
if filename in self._open_files: |
|
591 |
raise IOError("File already open: %s" %filename) |
|
592 |
||
593 |
if not re.compile("[rwa][+b]{0,2}").match(mode): |
|
594 |
raise IOError("invalid mode %s" %mode) |
|
595 |
||
596 |
## TODO: validate filename?
|
|
597 |
||
598 |
mode.replace("b",'') |
|
599 |
||
600 |
# initialise the file properly (truncate/create if required)
|
|
601 |
if mode[0] == 'w': |
|
602 |
self._files[filename] = '' |
|
603 |
self._modified_files.add(filename) |
|
604 |
elif filename not in self._files: |
|
605 |
if mode[0] == 'a': |
|
606 |
self._files[filename] = '' |
|
607 |
self._modified_files.add(filename) |
|
608 |
else: |
|
609 |
raise IOError(2, "Access to file denied: %s" %filename) |
|
610 |
||
611 |
# for append mode, remember the existing data
|
|
612 |
if mode[0] == 'a': |
|
613 |
existing_data = self._files[filename] |
|
614 |
else: |
|
615 |
existing_data = "" |
|
616 |
||
617 |
# determine what operations are allowed
|
|
618 |
reading_ok = (len(mode) == 2 or mode[0] == 'r') |
|
619 |
writing_ok = (len(mode) == 2 or mode[0] in 'wa') |
|
620 |
||
621 |
# for all writing modes, start off with blank file
|
|
622 |
if mode[0] == 'w': |
|
623 |
initial_data = '' |
|
624 |
else: |
|
625 |
initial_data = self._files[filename] |
|
626 |
||
627 |
file_object = TestStringIO(initial_data, filename, self, reading_ok, writing_ok, existing_data) |
|
628 |
self._open_files[filename] = file_object |
|
629 |
||
630 |
return file_object |
|
631 |
||
632 |
def flush_all(self): |
|
633 |
""" Flush all open files
|
|
634 |
"""
|
|
635 |
for file_object in self._open_files.values(): |
|
636 |
file_object.flush() |
|
637 |
||
638 |
def updatefile(self,filename, data): |
|
639 |
""" Callback function used by an open file to inform when it has been updated.
|
|
640 |
"""
|
|
641 |
if filename in self._open_files: |
|
642 |
self._files[filename] = data |
|
643 |
if self._open_files[filename].is_modified(): |
|
644 |
self._modified_files.add(filename) |
|
645 |
else: |
|
646 |
raise IOError(2, "Access to file denied: %s" %filename) |
|
647 |
||
648 |
def closefile(self, filename): |
|
649 |
""" Callback function used by an open file to inform when it has been closed.
|
|
650 |
"""
|
|
651 |
if filename in self._open_files: |
|
652 |
del self._open_files[filename] |
|
653 |
||
654 |
def get_modified_files(self): |
|
655 |
"""" A subset of the filespace containing only those files which have been
|
|
656 |
modified
|
|
657 |
"""
|
|
658 |
modified_files = {} |
|
659 |
for filename in self._modified_files: |
|
660 |
modified_files[filename] = self._files[filename] |
|
661 |
||
662 |
return modified_files |
|
663 |
||
664 |
def get_open_files(self): |
|
665 |
" Return the names of all open files "
|
|
666 |
return self._open_files.keys() |
|
667 |
||
668 |
def copy(self): |
|
669 |
""" Return a copy of the current filespace.
|
|
670 |
Only the files are copied, not the modified or open file lists.
|
|
671 |
"""
|
|
672 |
self.flush_all() |
|
673 |
return TestFilespace(self._files) |
|
674 |
||
675 |
class TestStringIO(StringIO.StringIO): |
|
676 |
"""
|
|
677 |
A subclass of StringIO which acts as a file in our dummy file system
|
|
678 |
"""
|
|
679 |
def __init__(self, string, filename, filespace, reading_ok, writing_ok, existing_data): |
|
680 |
""" Initialise with the filedata, file name and infomation on what ops are
|
|
681 |
acceptable """
|
|
682 |
StringIO.StringIO.__init__(self, string) |
|
683 |
self._filename = filename |
|
684 |
self._filespace = filespace |
|
685 |
self._reading_ok = reading_ok |
|
686 |
self._writing_ok = writing_ok |
|
687 |
self._existing_data = existing_data |
|
688 |
self._modified = False |
|
689 |
self._open = True |
|
690 |
||
691 |
# Override all standard file ops. Make sure that they are valid with the given
|
|
692 |
# permissions and if so then call the corresponding method in StringIO
|
|
693 |
||
694 |
def read(self, *args): |
|
695 |
if not self._reading_ok: |
|
696 |
raise IOError(9, "Bad file descriptor") |
|
697 |
else: |
|
698 |
return StringIO.StringIO.read(self, *args) |
|
699 |
||
700 |
def readline(self, *args): |
|
701 |
if not self._reading_ok: |
|
702 |
raise IOError(9, "Bad file descriptor") |
|
703 |
else: |
|
704 |
return StringIO.StringIO.readline(self, *args) |
|
705 |
||
706 |
def readlines(self, *args): |
|
707 |
if not self._reading_ok: |
|
708 |
raise IOError(9, "Bad file descriptor") |
|
709 |
else: |
|
710 |
return StringIO.StringIO.readlines(self, *args) |
|
711 |
||
712 |
def seek(self, *args): |
|
713 |
if not self._reading_ok: |
|
714 |
raise IOError(9, "Bad file descriptor") |
|
715 |
else: |
|
716 |
return StringIO.StringIO.seek(self, *args) |
|
717 |
||
718 |
def truncate(self, *args): |
|
719 |
self._modified = True |
|
720 |
if not self._writing_ok: |
|
721 |
raise IOError(9, "Bad file descriptor") |
|
722 |
else: |
|
723 |
return StringIO.StringIO.truncate(self, *args) |
|
724 |
||
725 |
def write(self, *args): |
|
726 |
self._modified = True |
|
727 |
if not self._writing_ok: |
|
728 |
raise IOError(9, "Bad file descriptor") |
|
729 |
else: |
|
730 |
return StringIO.StringIO.write(self, *args) |
|
731 |
||
732 |
def writelines(self, *args): |
|
733 |
self._modified = True |
|
734 |
if not self._writing_ok: |
|
735 |
raise IOError(9, "Bad file descriptor") |
|
736 |
else: |
|
737 |
return StringIO.StringIO.writelines(self, *args) |
|
738 |
||
739 |
def is_modified(self): |
|
740 |
" Return true if the file has been written to, or truncated"
|
|
741 |
return self._modified |
|
742 |
||
743 |
def flush(self): |
|
744 |
" Update the contents of the filespace with the new data "
|
|
745 |
self._filespace.updatefile(self._filename, self._existing_data+self.getvalue()) |
|
746 |
return StringIO.StringIO.flush(self) |
|
747 |
||
748 |
def close(self): |
|
749 |
" Flush the file and close it "
|
|
750 |
self.flush() |
|
751 |
self._filespace.closefile(self._filename) |
|
752 |
return StringIO.StringIO.close(self) |
|
753 |
||
754 |
##def get_function(filename, function_name):
|
|
755 |
## import compiler
|
|
756 |
## mod = compiler.parseFile(filename)
|
|
757 |
## for node in mod.node.nodes:
|
|
758 |
## if isinstance(node, compiler.ast.Function) and node.name == function_name:
|
|
759 |
## return node
|
|
760 |
##
|