« back to all changes in this revision
Viewing changes to www/apps/tutorial/test/TestFramework.py
- browse files at revision 270
- compare with another revision
- compare with revision 31
- download diff
- download tarball
- view history from revision 270
- Committer: dilshan_a
- Date: 2008-01-23 04:54:10 UTC
- Revision ID: svn-v3-trunk0:2b9c9e99-6f39-0410-b283-7f802c844ae2:trunk:270
Initial check in for test framework, and example problems
- files added:
- test/test_framework
- www/apps/tutorial/test
added
removed
www/apps/tutorial/test/TestFramework.py
1
# define custom exceptions
2
# use exceptions for all errors found in testing
3
4
import sys, StringIO, copy
5
6
# student error
7
class FunctionNotFoundError(Exception):
8
"""This error is returned when a function was expected in student
9
code but was not found"""
10
def __init__(self, function_name):
11
self.function_name = function_name
12
13
def __str__(self):
14
return "Function " + self.function_name + " not found"
15
16
# author error
17
class TestCreationError(Exception):
18
"""An error occured while creating the test suite or one of its components"""
19
def __init__(self, reason):
20
self._reason = reason
21
22
def __str__(self):
23
return self._reason
24
25
# author error
26
class SolutionError(Exception):
27
"""Error in the provided solution"""
28
def __init__(self, exc_info):
29
cla, exc, trbk = exc_info
30
self.name = cla.__name__
31
self._detail = str(exc)
32
33
def __str__(self):
34
return "Error running solution: %s" %str(self._detail)
35
36
# author error
37
class TestError(Exception):
38
"""Runtime error in the testing framework outside of the provided or student code"""
39
def __init__(self, exc_info):
40
cla, exc, trbk = exc_info
41
self.name = cla.__name__
42
self._detail = str(exc)
43
44
def __str__(self):
45
return "Error testing solution against attempt: %s" %str(self._detail)
46
47
# student error
48
class AttemptError(Exception):
49
"""Runtime error in the student code"""
50
def __init__(self, exc_info):
51
cla, exc, trbk = exc_info
52
self._name = cla.__name__
53
self._detail = str(exc)
54
55
def is_critical(self):
56
if ( self._name == 'FunctionNotFoundError'
57
or self._name == 'SyntaxError'
58
or self._name == 'IndentationError'):
59
return True
60
else:
61
return False
62
63
def __str__(self):
64
return self._name + " - " + str(self._detail)
65
66
class TestCasePart:
67
"""
68
A part of a test case which compares a subset of the input files or file streams.
69
This can be done either with a comparision function, or by comparing directly, after
70
applying normalisations.
71
"""
72
# how to make this work? atm they seem to get passed the class as a first arg
73
ident =lambda x: x
74
ignore = lambda x: None
75
match = lambda x,y: x==y
76
always_match = lambda x,y: True
77
true = lambda *x: True
78
false = lambda *x: False
79
80
def __init__(self, desc, default='match'):
81
"""Initialise with a description and a default behavior for output
82
If default is match, unspecified files are matched exactly
83
If default is ignore, unspecified files are ignored
84
The default default is match.
85
"""
86
self._desc = desc
87
self._default = default
88
if default == 'ignore':
89
self._default_func = lambda *x: True
90
else:
91
self._default_func = lambda x,y: x==y
92
93
self._file_tests = {}
94
self._stdout_test = ('check', self._default_func)
95
self._stderr_test = ('check', self._default_func)
96
self._result_test = ('check', self._default_func)
97
98
def get_description(self):
99
"Getter for description"
100
return self._desc
101
102
def _set_default_function(self, function, test_type):
103
""""Ensure test type is valid and set function to a default
104
if not specified"""
105
106
if test_type not in ['norm', 'check']:
107
raise TestCreationError("Invalid test type in %s" %self._desc)
108
109
if function == '':
110
if test_type == 'norm': function = lambda x: x
111
else: function = lambda x,y: x==y
112
113
return function
114
115
def _validate_function(self, function, included_code):
116
"""Create a function object from the given string.
117
If a valid function object cannot be created, raise and error.
118
"""
119
if not callable(function):
120
try:
121
exec "__f__ = %s" %function in included_code
122
except:
123
raise TestCreationError("Invalid function %s" %function)
124
125
f = included_code['__f__']
126
127
if not callable(f):
128
raise TestCreationError("Invalid function %s" %function)
129
else:
130
f = function
131
132
return f
133
134
def validate_functions(self, included_code):
135
"""Ensure all functions used by the test cases exist and are callable.
136
Also covert their string representations to function objects.
137
This can only be done once all the include code has been specified.
138
"""
139
(test_type, function) = self._stdout_test
140
self._stdout_test = (test_type, self._validate_function(function, included_code))
141
142
(test_type, function) = self._stderr_test
143
self._stderr_test = (test_type, self._validate_function(function, included_code))
144
145
for filename, (test_type, function) in self._file_tests.items():
146
self._file_tests[filename] = (test_type, self._validate_function(function, included_code))
147
148
def add_result_test(self, function, test_type='norm'):
149
"Test part that compares function return values"
150
function = self._set_default_function(function, test_type)
151
self._result_test = (test_type, function)
152
153
154
def add_stdout_test(self, function, test_type='norm'):
155
"Test part that compares stdout"
156
function = self._set_default_function(function, test_type)
157
self._stdout_test = (test_type, function)
158
159
160
def add_stderr_test(self, function, test_type='norm'):
161
"Test part that compares stderr"
162
function = self._set_default_function(function, test_type)
163
self._stderr_test = (test_type, function)
164
165
def add_file_test(self, filename, function, test_type='norm'):
166
"Test part that compares the contents of a specified file"
167
function = self._set_default_function(function, test_type)
168
self._file_tests[filename] = (test_type, function)
169
170
def _check_output(self, solution_output, attempt_output, test_type, f):
171
"""Compare solution output and attempt output using the
172
specified comparision function.
173
"""
174
# converts unicode to string
175
if type(solution_output) == unicode:
176
solution_output = str(solution_output)
177
if type(attempt_output) == unicode:
178
attempt_output = str(attempt_output)
179
180
if test_type == 'norm':
181
return f(solution_output) == f(attempt_output)
182
else:
183
return f(solution_output, attempt_output)
184
185
def run(self, solution_data, attempt_data):
186
"""Run the tests to compare the solution and attempt data
187
Returns the empty string is the test passes, or else an error message.
188
"""
189
190
# check function return value (None for scripts)
191
(test_type, f) = self._result_test
192
if not self._check_output(solution_data['result'], attempt_data['result'], test_type, f):
193
return 'function return value does not match'
194
195
# check stdout
196
(test_type, f) = self._stdout_test
197
if not self._check_output(solution_data['stdout'], attempt_data['stdout'], test_type, f):
198
return 'stdout does not match'
199
200
#check stderr
201
(test_type, f) = self._stderr_test
202
if not self._check_output(solution_data['stderr'], attempt_data['stderr'], test_type, f):
203
return 'stderr does not match'
204
205
206
solution_files = solution_data['modified_files']
207
attempt_files = attempt_data['modified_files']
208
209
# check files indicated by test
210
for (filename, (test_type, f)) in self._file_tests.items():
211
if filename not in solution_files:
212
raise SolutionError('File %s not found' %filename)
213
elif filename not in attempt_files:
214
return filename + ' not found'
215
elif not self._check_output(solution_files[filename], attempt_files[filename], test_type, f):
216
return filename + ' does not match'
217
218
if self._default == 'ignore':
219
return ''
220
221
# check files found in solution, but not indicated by test
222
for filename in [f for f in solution_files if f not in self._file_tests]:
223
if filename not in attempt_files:
224
return filename + ' not found'
225
elif not self._check_output(solution_files[filename], attempt_files[filename], 'match', lambda x,y: x==y):
226
return filename + ' does not match'
227
228
# check if attempt has any extra files
229
for filename in [f for f in attempt_files if f not in solution_files]:
230
return "Unexpected file found: " + filename
231
232
# Everything passed with no problems
233
return ''
234
235
class TestCase:
236
"""
237
A set of tests with a common inputs
238
"""
239
def __init__(self, name='', function=None, stdin='', filespace={}, global_space={}):
240
"""Initialise with name and optionally, a function to test (instead of the entire script)
241
The inputs stdin, the filespace and global variables can also be specified at
242
initialisation, but may also be set later.
243
"""
244
self._name = name
245
246
if function == '': function = None
247
self._function = function
248
self._list_args = []
249
self._keyword_args = {}
250
251
# stdin must have a newline at the end for raw_input to work properly
252
if stdin[-1:] != '\n': stdin += '\n'
253
254
self._stdin = stdin
255
self._filespace = TestFilespace(filespace)
256
self._global_space = global_space
257
self._parts = []
258
259
def set_stdin(self, stdin):
260
""" Set the given string as the stdin for this test case"""
261
self._stdin = stdin
262
263
def add_file(self, filename, data):
264
""" Insert the given filename-data pair into the filespace for this test case"""
265
self._filespace.add_file(filename, data)
266
267
def add_variable(self, variable, value):
268
""" Add the given varibale-value pair to the initial global environment
269
for this test case.
270
Throw and exception if thevalue cannot be paresed.
271
"""
272
try:
273
self._global_space[variable] = eval(value)
274
except:
275
raise TestCreationError("Invalid value for variable %s: %s" %(varible, value))
276
277
def add_arg(self, value, name=None):
278
""" Add a value to the argument list. This only applies when testing functions.
279
By default arguments are not named, but if they are, they become keyword arguments.
280
"""
281
try:
282
if name == None or name == '':
283
self._list_args.append(eval(value))
284
else:
285
self._keyword_args[name] = value
286
except:
287
raise TestCreationError("Invalid value for function argument: %s" %value)
288
289
def add_part(self, test_part):
290
""" Add a TestPart to this test case"""
291
self._parts.append(test_part)
292
293
def validate_functions(self, included_code):
294
""" Validate all the functions in each part in this test case
295
This can only be done once all the include code has been specified.
296
"""
297
for part in self._parts:
298
part.validate_functions(included_code)
299
300
def get_name(self):
301
""" Get the name of the test case """
302
return self._name
303
304
def run(self, solution, attempt_file):
305
""" Run the solution and the attempt with the inputs specified for this test case.
306
Then pass the outputs to each test part and collate the results.
307
"""
308
# Run solution
309
try:
310
global_space_copy = copy.deepcopy(self._global_space)
311
solution_data = self._execstring(solution, global_space_copy)
312
313
# if we are just testing a function
314
if not self._function == None:
315
if self._function not in global_space_copy:
316
raise FunctionNotFoundError(self._function)
317
solution_data = self._run_function(lambda: global_space_copy[self._function](*self._list_args, **self._keyword_args))
318
319
except:
320
raise SolutionError(sys.exc_info())
321
322
# Run student attempt
323
try:
324
global_space_copy = copy.deepcopy(self._global_space)
325
attempt_data = self._execfile(attempt_file, global_space_copy)
326
327
# if we are just testing a function
328
if not self._function == None:
329
if self._function not in global_space_copy:
330
raise FunctionNotFoundError(self._function)
331
attempt_data = self._run_function(lambda: global_space_copy[self._function](*self._list_args, **self._keyword_args))
332
except:
333
raise AttemptError(sys.exc_info())
334
335
results = []
336
337
# generate results
338
for test_part in self._parts:
339
result = test_part.run(solution_data, attempt_data)
340
341
results.append((test_part.get_description(), result))
342
343
return results
344
345
def _execfile(self, filename, global_space):
346
""" Execute the file given by 'filename' in global_space, and return the outputs. """
347
self._initialise_global_space(global_space)
348
data = self._run_function(lambda: execfile(filename, global_space))
349
return data
350
351
def _execstring(self, string, global_space):
352
""" Execute the given string in global_space, and return the outputs. """
353
self._initialise_global_space(global_space)
354
# _run_function handles tuples in a special way
355
data = self._run_function((string, global_space))
356
return data
357
358
def _initialise_global_space(self, global_space):
359
""" Modify the provided global_space so that file, open and raw_input are redefined
360
to use our methods instead.
361
"""
362
self._current_filespace_copy = self._filespace.copy()
363
global_space['file'] = lambda filename, mode='r', bufsize=-1: self._current_filespace_copy.openfile(filename, mode)
364
global_space['open'] = global_space['file']
365
global_space['raw_input'] = lambda x=None: raw_input()
366
return global_space
367
368
def _run_function(self, function):
369
""" Run the provided function with the provided stdin, capturing stdout and stderr
370
and the return value.
371
Return all the output data.
372
"""
373
import sys, StringIO
374
sys_stdout, sys_stdin, sys_stderr = sys.stdout, sys.stdin, sys.stderr
375
376
output_stream, input_stream, error_stream = StringIO.StringIO(), StringIO.StringIO(self._stdin), StringIO.StringIO()
377
sys.stdout, sys.stdin, sys.stderr = output_stream, input_stream, error_stream
378
379
try:
380
if type(function) == tuple:
381
# very hackish... exec can't be put into a lambda function!
382
# or even with eval
383
exec(function[0], function[1])
384
result = None
385
else:
386
result = function()
387
except:
388
sys.stdout, sys.stdin, sys.stderr = sys_stdout, sys_stdin, sys_stderr
389
raise
390
391
sys.stdout, sys.stdin, sys.stderr = sys_stdout, sys_stdin, sys_stderr
392
393
self._current_filespace_copy.flush_all()
394
395
return {'result': result,
396
'stdout': output_stream.getvalue(),
397
'stderr': output_stream.getvalue(),
398
'modified_files': self._current_filespace_copy.get_modified_files()}
399
400
class TestSuite:
401
"""
402
The complete collection of test cases for a given problem
403
"""
404
def __init__(self, name, solution=None):
405
"""Initialise with the name of the test suite (the problem name) and the solution.
406
The solution may be specified later.
407
"""
408
self._solution = solution
409
self._name = name
410
self._tests = []
411
self.add_include_code("")
412
413
def add_solution(self, solution):
414
" Specifiy the solution script for this problem "
415
self._solution = solution
416
417
def has_solution(self):
418
" Returns true if a soltion has been provided "
419
return self._solution != None
420
421
def add_include_code(self, include_code = ''):
422
""" Add include code that may be used by the test cases during
423
comparison of outputs.
424
"""
425
426
# if empty, make sure it can still be executed
427
if include_code == "":
428
include_code = "pass"
429
self._include_code = str(include_code)
430
431
include_space = {}
432
try:
433
exec self._include_code in include_space
434
except:
435
raise TestCreationError("Bad include code")
436
437
self._include_space = include_space
438
439
def add_case(self, test_case):
440
""" Add a TestCase, then validate all functions inside test case
441
now that the include code is known
442
"""
443
self._tests.append(test_case)
444
test_case.validate_functions(self._include_space)
445
446
def run_tests(self, attempt_file):
447
" Run all test cases and collate the results "
448
449
results = []
450
451
for test in self._tests:
452
test_results = []
453
try:
454
for (name, result) in test.run(self._solution, attempt_file):
455
if result == '':
456
disp = "Passed: %s" %name
457
else:
458
disp = "Failed: %s, %s" %(name,result)
459
test_results.append(disp)
460
except AttemptError, e:
461
test_results.append("Error running submitted script: %s" %str(e))
462
results.append((test.get_name(), test_results))
463
return (self._name, results)
464
465
class TestFilespace:
466
"""
467
Our dummy file system which is accessed by code being tested.
468
Implemented as a dictionary which maps filenames to strings
469
"""
470
def __init__(self, files={}):
471
"Initialise, optionally with filename-filedata pairs"
472
473
# dict mapping files to strings
474
self._files = {}
475
self._files.update(files)
476
# set of file names
477
self._modified_files = set([])
478
# dict mapping files to stringIO objects
479
self._open_files = {}
480
481
def add_file(self, filename, data):
482
" Add a file to the filespace "
483
self._files[filename] = data
484
485
def openfile(self, filename, mode='r'):
486
""" Open a file from the filespace with the given mode.
487
Return a StringIO subclass object with the file contents.
488
"""
489
import re
490
491
if filename in self._open_files:
492
raise IOError("File already open: %s" %filename)
493
494
if not re.compile("[rwa][+b]{0,2}").match(mode):
495
raise IOError("invalid mode %s" %mode)
496
497
## TODO: validate filename?
498
499
mode.replace("b",'')
500
501
# initialise the file properly (truncate/create if required)
502
if mode[0] == 'w':
503
self._files[filename] = ''
504
self._modified_files.add(filename)
505
elif filename not in self._files:
506
if mode[0] == 'a':
507
self._files[filename] = ''
508
self._modified_files.add(filename)
509
else:
510
raise IOError(2, "Access to file denied: %s" %filename)
511
512
# for append mode, remember the existing data
513
if mode[0] == 'a':
514
existing_data = self._files[filename]
515
else:
516
existing_data = ""
517
518
# determine what operations are allowed
519
reading_ok = (len(mode) == 2 or mode[0] == 'r')
520
writing_ok = (len(mode) == 2 or mode[0] in 'wa')
521
522
# for all writing modes, start off with blank file
523
if mode[0] == 'w':
524
initial_data = ''
525
else:
526
initial_data = self._files[filename]
527
528
file_object = TestStringIO(initial_data, filename, self, reading_ok, writing_ok, existing_data)
529
self._open_files[filename] = file_object
530
531
return file_object
532
533
def flush_all(self):
534
""" Flush all open files
535
"""
536
for file_object in self._open_files.values():
537
file_object.flush()
538
539
def updatefile(self,filename, data):
540
""" Callback function used by an open file to inform when it has been updated.
541
"""
542
if filename in self._open_files:
543
self._files[filename] = data
544
if self._open_files[filename].is_modified():
545
self._modified_files.add(filename)
546
else:
547
raise IOError(2, "Access to file denied: %s" %filename)
548
549
def closefile(self, filename):
550
""" Callback function used by an open file to inform when it has been closed.
551
"""
552
if filename in self._open_files:
553
del self._open_files[filename]
554
555
def get_modified_files(self):
556
"""" A subset of the filespace containing only those files which have been
557
modified
558
"""
559
modified_files = {}
560
for filename in self._modified_files:
561
modified_files[filename] = self._files[filename]
562
563
return modified_files
564
565
def get_open_files(self):
566
" Return the names of all open files "
567
return self._open_files.keys()
568
569
def copy(self):
570
""" Return a copy of the current filespace.
571
Only the files are copied, not the modified or open file lists.
572
"""
573
self.flush_all()
574
return TestFilespace(self._files)
575
576
class TestStringIO(StringIO.StringIO):
577
"""
578
A subclass of StringIO which acts as a file in our dummy file system
579
"""
580
def __init__(self, string, filename, filespace, reading_ok, writing_ok, existing_data):
581
""" Initialise with the filedata, file name and infomation on what ops are
582
acceptable """
583
StringIO.StringIO.__init__(self, string)
584
self._filename = filename
585
self._filespace = filespace
586
self._reading_ok = reading_ok
587
self._writing_ok = writing_ok
588
self._existing_data = existing_data
589
self._modified = False
590
self._open = True
591
592
# Override all standard file ops. Make sure that they are valid with the given
593
# permissions and if so then call the corresponding method in StringIO
594
595
def read(self, *args):
596
if not self._reading_ok:
597
raise IOError(9, "Bad file descriptor")
598
else:
599
return StringIO.StringIO.read(self, *args)
600
601
def readline(self, *args):
602
if not self._reading_ok:
603
raise IOError(9, "Bad file descriptor")
604
else:
605
return StringIO.StringIO.readline(self, *args)
606
607
def readlines(self, *args):
608
if not self._reading_ok:
609
raise IOError(9, "Bad file descriptor")
610
else:
611
return StringIO.StringIO.readlines(self, *args)
612
613
def seek(self, *args):
614
if not self._reading_ok:
615
raise IOError(9, "Bad file descriptor")
616
else:
617
return StringIO.StringIO.seek(self, *args)
618
619
def truncate(self, *args):
620
self._modified = True
621
if not self._writing_ok:
622
raise IOError(9, "Bad file descriptor")
623
else:
624
return StringIO.StringIO.truncate(self, *args)
625
626
def write(self, *args):
627
self._modified = True
628
if not self._writing_ok:
629
raise IOError(9, "Bad file descriptor")
630
else:
631
return StringIO.StringIO.write(self, *args)
632
633
def writelines(self, *args):
634
self._modified = True
635
if not self._writing_ok:
636
raise IOError(9, "Bad file descriptor")
637
else:
638
return StringIO.StringIO.writelines(self, *args)
639
640
def is_modified(self):
641
" Return true if the file has been written to, or truncated"
642
return self._modified
643
644
def flush(self):
645
" Update the contents of the filespace with the new data "
646
self._filespace.updatefile(self._filename, self._existing_data+self.getvalue())
647
return StringIO.StringIO.flush(self)
648
649
def close(self):
650
" Flush the file and close it "
651
self.flush()
652
self._filespace.closefile(self._filename)
653
return StringIO.StringIO.close(self)
654
655
##def get_function(filename, function_name):
656
## import compiler
657
## mod = compiler.parseFile(filename)
658
## for node in mod.node.nodes:
659
## if isinstance(node, compiler.ast.Function) and node.name == function_name:
660
## return node
661
##
Loggerhead is a web-based interface for Breezy
Version: 2.0.1