23
23
# This is a command-line application, for use by the administrator.
24
# This program is a frontend for the modules in the setup packages that
25
# build and install IVLE in separate steps.
24
# This program configures, builds and installs IVLE in three separate steps.
26
25
# It is called with at least one argument, which specifies which operation to
28
# setup.py listmake (for developer use only)
29
# Recurses through the source tree and builds a list of all files which should
30
# be copied upon installation. This should be run by the developer before
31
# cutting a distribution, and the listfile it generates should be included in
32
# the distribution, avoiding the administrator having to run it.
34
# setup.py conf [args]
35
# Configures IVLE with machine-specific details, most notably, various paths.
36
# Either prompts the administrator for these details or accepts them as
38
# Creates www/conf/conf.py and trampoline/conf.h.
41
# Compiles all files and sets up a jail template in the source directory.
43
# Compiles (GCC) trampoline/trampoline.c to trampoline/trampoline.
45
# Creates standard subdirs inside the jail, eg bin, opt, home, tmp.
46
# Copies console/ to a location within the jail.
47
# Copies OS programs and files to corresponding locations within the jail
48
# (eg. python and Python libs, ld.so, etc).
49
# Generates .pyc files for all the IVLE .py files.
51
# setup.py install [--nojail] [--dry|n]
53
# Create target install directory ($target).
55
# Copy trampoline/trampoline to $target/bin.
56
# chown and chmod the installed trampoline.
57
# Copy www/ to $target.
58
# Copy jail/ to jails template directory (unless --nojail specified).
71
# Import modules from the website is tricky since they're in the www
73
sys.path.append(os.path.join(os.getcwd(), 'www'))
75
import common.makeuser
77
# Operating system files to copy over into the jail.
78
# These will be copied from the given place on the OS file system into the
79
# same place within the jail.
82
'/lib/tls/i686/cmov/libc.so.6',
83
'/lib/tls/i686/cmov/libdl.so.2',
84
'/lib/tls/i686/cmov/libm.so.6',
85
'/lib/tls/i686/cmov/libpthread.so.0',
86
'/lib/tls/i686/cmov/libutil.so.1',
89
# These 2 files do not exist in Ubuntu
90
#'/etc/ld.so.preload',
91
#'/etc/ld.so.nohwcap',
98
# Needed by matplotlib
99
'/usr/lib/i686/cmov/libssl.so.0.9.8',
100
'/usr/lib/i686/cmov/libcrypto.so.0.9.8',
101
'/lib/tls/i686/cmov/libnsl.so.1',
102
'/usr/lib/libz.so.1',
103
'/usr/lib/atlas/liblapack.so.3',
104
'/usr/lib/atlas/libblas.so.3',
105
'/usr/lib/libg2c.so.0',
106
'/usr/lib/libstdc++.so.6',
107
'/usr/lib/libfreetype.so.6',
108
'/usr/lib/libpng12.so.0',
109
'/usr/lib/libBLT.2.4.so.8.4',
110
'/usr/lib/libtk8.4.so.0',
111
'/usr/lib/libtcl8.4.so.0',
112
'/usr/lib/tcl8.4/init.tcl',
113
'/usr/lib/libX11.so.6',
114
'/usr/lib/libXau.so.6',
115
'/usr/lib/libXdmcp.so.6',
116
'/lib/libgcc_s.so.1',
119
# Symlinks to make within the jail. Src mapped to dst.
121
'python2.5': 'jail/usr/bin/python',
123
# Trees to copy. Src mapped to dst (these will be passed to action_copytree).
125
'/usr/lib/python2.5': 'jail/usr/lib/python2.5',
126
'/usr/share/matplotlib': 'jail/usr/share/matplotlib',
127
'/etc/ld.so.conf.d': 'jail/etc/ld.so.conf.d',
130
# Try importing existing conf, but if we can't just set up defaults
131
# The reason for this is that these settings are used by other phases
132
# of setup besides conf, so we need to know them.
133
# Also this allows you to hit Return to accept the existing value.
135
confmodule = __import__("www/conf/conf")
137
root_dir = confmodule.root_dir
141
ivle_install_dir = confmodule.ivle_install_dir
143
ivle_install_dir = "/opt/ivle"
145
public_host = confmodule.public_host
147
public_host = "public.localhost"
149
jail_base = confmodule.jail_base
151
jail_base = "/home/informatics/jails"
153
subjects_base = confmodule.subjects_base
155
subjects_base = "/home/informatics/subjects"
157
# Just set reasonable defaults
159
ivle_install_dir = "/opt/ivle"
160
public_host = "public.localhost"
161
jail_base = "/home/informatics/jails"
162
subjects_base = "/home/informatics/subjects"
166
# Try importing install_list, but don't fail if we can't, because listmake can
167
# function without it.
173
# Mime types which will automatically be placed in the list by listmake.
174
# Note that listmake is not intended to be run by the final user (the system
175
# administrator who installs this), so the developers can customize the list
176
# as necessary, and include it in the distribution.
177
listmake_mimetypes = ['text/x-python', 'text/html',
178
'application/x-javascript', 'application/javascript',
179
'text/css', 'image/png']
181
# Main function skeleton from Guido van Rossum
182
# http://www.artima.com/weblogs/viewpost.jsp?thread=4829
33
184
def main(argv=None):
56
oper_func = call_operator(operation)
57
return oper_func(argv[2:])
61
print """Usage: python setup.py operation [options]
67
For help and options for a specific operation use 'help [operation]'."""
70
oper_func = call_operator(operator)
71
oper_func(['operator','--help'])
73
def call_operator(operation):
207
# Disallow run as root unless installing
208
if (operation != 'install' and operation != 'updatejails'
209
and os.geteuid() == 0):
210
print >>sys.stderr, "I do not want to run this stage as root."
211
print >>sys.stderr, "Please run as a normal user."
74
213
# Call the requested operation's function
78
'build' : setup.build.build,
79
'install' : setup.install.install,
219
'listmake' : listmake,
221
'updatejails' : updatejails,
82
224
print >>sys.stderr, (
83
225
"""Invalid operation '%s'. Try python setup.py help."""
228
return oper_func(argv[2:])
230
# Operation functions
234
print """Usage: python setup.py operation [args]
235
Operation (and args) can be:
237
listmake (developer use only)
240
install [--nojail] [-n|--dry]
244
print """Usage: python setup.py help [operation]"""
249
if operation == 'help':
250
print """python setup.py help [operation]
251
Prints the usage message or detailed help on an operation, then exits."""
252
elif operation == 'listmake':
253
print """python setup.py listmake
254
(For developer use only)
255
Recurses through the source tree and builds a list of all files which should
256
be copied upon installation. This should be run by the developer before
257
cutting a distribution, and the listfile it generates should be included in
258
the distribution, avoiding the administrator having to run it."""
259
elif operation == 'conf':
260
print """python setup.py conf [args]
261
Configures IVLE with machine-specific details, most notably, various paths.
262
Either prompts the administrator for these details or accepts them as
263
command-line args. Will be interactive only if there are no arguments given.
264
Takes defaults from existing conf file if it exists.
266
To run IVLE out of the source directory (allowing development without having
267
to rebuild/install), just provide ivle_install_dir as the IVLE trunk
268
directory, and run build/install one time.
270
Creates www/conf/conf.py and trampoline/conf.h.
279
As explained in the interactive prompt or conf.py.
281
elif operation == 'build':
282
print """python -O setup.py build [--dry|-n]
283
Compiles all files and sets up a jail template in the source directory.
284
-O is recommended to cause compilation to be optimised.
286
Compiles (GCC) trampoline/trampoline.c to trampoline/trampoline.
288
Creates standard subdirs inside the jail, eg bin, opt, home, tmp.
289
Copies console/ to a location within the jail.
290
Copies OS programs and files to corresponding locations within the jail
291
(eg. python and Python libs, ld.so, etc).
292
Generates .pyc or .pyo files for all the IVLE .py files.
294
--dry | -n Print out the actions but don't do anything."""
295
elif operation == 'install':
296
print """sudo python setup.py install [--nojail] [--dry|-n]
298
Create target install directory ($target).
300
Copy trampoline/trampoline to $target/bin.
301
chown and chmod the installed trampoline.
302
Copy www/ to $target.
303
Copy jail/ to jails template directory (unless --nojail specified).
305
--nojail Do not copy the jail.
306
--dry | -n Print out the actions but don't do anything."""
307
elif operation == 'updatejails':
308
print """sudo python setup.py updatejails [--dry|-n]
310
Copy jail/ to each subdirectory in jails directory.
312
--dry | -n Print out the actions but don't do anything."""
314
print >>sys.stderr, (
315
"""Invalid operation '%s'. Try python setup.py help."""
320
# We build two separate lists, by walking www and console
321
list_www = build_list_py_files('www')
322
list_console = build_list_py_files('console')
323
# Make sure that the files generated by conf are in the list
324
# (since listmake is typically run before conf)
325
if "www/conf/conf.py" not in list_www:
326
list_www.append("www/conf/conf.py")
327
# Make sure that console/python-console is in the list
328
if "console/python-console" not in list_console:
329
list_console.append("console/python-console")
330
# Write these out to a file
332
# the files that will be created/overwritten
333
listfile = os.path.join(cwd, "install_list.py")
336
file = open(listfile, "w")
338
file.write("""# IVLE Configuration File
340
# Provides lists of all files to be installed by `setup.py install' from
341
# certain directories.
342
# Note that any files with the given filename plus 'c' or 'o' (that is,
343
# compiled .pyc or .pyo files) will be copied as well.
345
# List of all installable files in www directory.
347
writelist_pretty(file, list_www)
349
# List of all installable files in console directory.
351
writelist_pretty(file, list_console)
354
except IOError, (errno, strerror):
355
print "IO error(%s): %s" % (errno, strerror)
358
print "Successfully wrote install_list.py"
361
print ("You may modify the set of installable files before cutting the "
368
def build_list_py_files(dir):
369
"""Builds a list of all py files found in a directory and its
370
subdirectories. Returns this as a list of strings."""
372
for (dirpath, dirnames, filenames) in os.walk(dir):
373
# Exclude directories beginning with a '.' (such as '.svn')
374
filter_mutate(lambda x: x[0] != '.', dirnames)
375
# All *.py files are added to the list
376
pylist += [os.path.join(dirpath, item) for item in filenames
377
if mimetypes.guess_type(item)[0] in listmake_mimetypes]
380
def writelist_pretty(file, list):
381
"""Writes a list one element per line, to a file."""
387
file.write(' %s,\n' % repr(elem))
391
global root_dir, ivle_install_dir, jail_base, subjects_base
392
global public_host, allowed_uids
393
# Set up some variables
396
# the files that will be created/overwritten
397
conffile = os.path.join(cwd, "www/conf/conf.py")
398
conf_hfile = os.path.join(cwd, "trampoline/conf.h")
400
# Get command-line arguments to avoid asking questions.
402
(opts, args) = getopt.gnu_getopt(args, "", ['root_dir=',
403
'ivle_install_dir=', 'jail_base=', 'allowed_uids='])
406
print >>sys.stderr, "Invalid arguments:", string.join(args, ' ')
410
# Interactive mode. Prompt the user for all the values.
412
print """This tool will create the following files:
415
prompting you for details about your configuration. The file will be
416
overwritten if it already exists. It will *not* install or deploy IVLE.
418
Please hit Ctrl+C now if you do not wish to do this.
419
""" % (conffile, conf_hfile)
421
# Get information from the administrator
422
# If EOF is encountered at any time during the questioning, just exit
425
root_dir = query_user(root_dir,
426
"""Root directory where IVLE is located (in URL space):""")
427
ivle_install_dir = query_user(ivle_install_dir,
428
'Root directory where IVLE will be installed (on the local file '
430
jail_base = query_user(jail_base,
431
"""Root directory where the jails (containing user files) are stored
432
(on the local file system):""")
433
subjects_base = query_user(subjects_base,
434
"""Root directory where the subject directories (containing worksheets
435
and other per-subject files) are stored (on the local file system):""")
436
public_host = query_user(public_host,
437
"""Hostname which will cause the server to go into "public mode",
438
providing login-free access to student's published work:""")
439
allowed_uids = query_user(allowed_uids,
440
"""UID of the web server process which will run IVLE.
441
Only this user may execute the trampoline. May specify multiple users as
442
a comma-separated list.
447
# Non-interactive mode. Parse the options.
448
if '--root_dir' in opts:
449
root_dir = opts['--root_dir']
450
if '--ivle_install_dir' in opts:
451
ivle_install_dir = opts['--ivle_install_dir']
452
if '--jail_base' in opts:
453
jail_base = opts['--jail_base']
454
if '--subjects_base' in opts:
455
jail_base = opts['--subjects_base']
456
if '--public_host' in opts:
457
public_host = opts['--public_host']
458
if '--allowed_uids' in opts:
459
allowed_uids = opts['--allowed_uids']
461
# Error handling on input values
463
allowed_uids = map(int, allowed_uids.split(','))
465
print >>sys.stderr, (
466
"Invalid UID list (%s).\n"
467
"Must be a comma-separated list of integers." % allowed_uids)
470
# Write www/conf/conf.py
473
conf = open(conffile, "w")
475
conf.write("""# IVLE Configuration File
477
# Miscellaneous application settings
480
# In URL space, where in the site is IVLE located. (All URLs will be prefixed
482
# eg. "/" or "/ivle".
485
# In the local file system, where IVLE is actually installed.
486
# This directory should contain the "www" and "bin" directories.
487
ivle_install_dir = "%s"
489
# The server goes into "public mode" if the browser sends a request with this
490
# host. This is for security reasons - we only serve public student files on a
491
# separate domain to the main IVLE site.
492
# Public mode does not use cookies, and serves only public content.
493
# Private mode (normal mode) requires login, and only serves files relevant to
494
# the logged-in user.
497
# In the local file system, where are the student/user file spaces located.
498
# The user jails are expected to be located immediately in subdirectories of
502
# In the local file system, where are the per-subject file spaces located.
503
# The individual subject directories are expected to be located immediately
504
# in subdirectories of this location.
506
""" % (root_dir, ivle_install_dir, public_host, jail_base, subjects_base))
509
except IOError, (errno, strerror):
510
print "IO error(%s): %s" % (errno, strerror)
513
print "Successfully wrote www/conf/conf.py"
515
# Write trampoline/conf.h
518
conf = open(conf_hfile, "w")
520
conf.write("""/* IVLE Configuration File
522
* Administrator settings required by trampoline.
523
* Note: trampoline will have to be rebuilt in order for changes to this file
527
/* In the local file system, where are the jails located.
528
* The trampoline does not allow the creation of a jail anywhere besides
529
* jail_base or a subdirectory of jail_base.
531
static const char* jail_base = "%s";
533
/* Which user IDs are allowed to run the trampoline.
534
* This list should be limited to the web server user.
535
* (Note that root is an implicit member of this list).
537
static const int allowed_uids[] = { %s };
538
""" % (jail_base, repr(allowed_uids)[1:-1]))
541
except IOError, (errno, strerror):
542
print "IO error(%s): %s" % (errno, strerror)
545
print "Successfully wrote trampoline/conf.h"
548
print "You may modify the configuration at any time by editing"
555
# Get "dry" variable from command line
556
(opts, args) = getopt.gnu_getopt(args, "n", ['dry'])
558
dry = '-n' in opts or '--dry' in opts
561
print "Dry run (no actions will be executed\n"
563
# Compile the trampoline
564
action_runprog('gcc', ['-Wall', '-o', 'trampoline/trampoline',
565
'trampoline/trampoline.c'], dry)
567
# Create the jail and its subdirectories
568
# Note: Other subdirs will be made by copying files
569
action_mkdir('jail', dry)
570
action_mkdir('jail/home', dry)
571
action_mkdir('jail/tmp', dry)
573
# Copy all console and operating system files into the jail
574
action_copylist(install_list.list_console, 'jail/opt/ivle', dry)
575
copy_os_files_jail(dry)
576
# Chmod the python console
577
action_chmod_x('jail/opt/ivle/console/python-console', dry)
580
# Compile .py files into .pyc or .pyo files
581
compileall.compile_dir('www', quiet=True)
582
compileall.compile_dir('console', quiet=True)
586
def copy_os_files_jail(dry):
587
"""Copies necessary Operating System files from their usual locations
588
into the jail/ directory of the cwd."""
589
# Currently source paths are configured for Ubuntu.
590
for filename in JAIL_FILES:
591
copy_file_to_jail(filename, dry)
592
for src, dst in JAIL_LINKS.items():
593
action_symlink(src, dst, dry)
594
for src, dst in JAIL_COPYTREES.items():
595
action_copytree(src, dst, dry)
597
def copy_file_to_jail(src, dry):
598
"""Copies a single file from an absolute location into the same location
599
within the jail. src must begin with a '/'. The jail will be located
600
in a 'jail' subdirectory of the current path."""
601
action_copyfile(src, 'jail' + src, dry)
604
# Get "dry" and "nojail" variables from command line
605
(opts, args) = getopt.gnu_getopt(args, "n", ['dry', 'nojail'])
607
dry = '-n' in opts or '--dry' in opts
608
nojail = '--nojail' in opts
611
print "Dry run (no actions will be executed\n"
613
if not dry and os.geteuid() != 0:
614
print >>sys.stderr, "Must be root to run install"
615
print >>sys.stderr, "(I need to chown some files)."
618
# Create the target (install) directory
619
action_mkdir(ivle_install_dir, dry)
621
# Create bin and copy the compiled files there
622
action_mkdir(os.path.join(ivle_install_dir, 'bin'), dry)
623
tramppath = os.path.join(ivle_install_dir, 'bin/trampoline')
624
action_copyfile('trampoline/trampoline', tramppath, dry)
625
# chown trampoline to root and set setuid bit
626
action_chown_setuid(tramppath, dry)
628
# Copy the www directory using the list
629
action_copylist(install_list.list_www, ivle_install_dir, dry)
632
# Copy the local jail directory built by the build action
633
# to the jails template directory (it will be used as a template
634
# for all the students' jails).
635
action_copytree('jail', os.path.join(jail_base, 'template'), dry)
637
# Append IVLE path to ivle.pth in python site packages
638
# (Unless it's already there)
639
ivle_pth = os.path.join(sys.prefix,
640
"lib/python2.5/site-packages/ivle.pth")
641
ivle_www = os.path.join(ivle_install_dir, "www")
642
write_ivle_pth = True
644
file = open(ivle_pth, 'r')
646
if line.strip() == ivle_www:
647
write_ivle_pth = False
649
except (IOError, OSError):
652
action_append(ivle_pth, ivle_www)
656
def updatejails(args):
657
# Get "dry" variable from command line
658
(opts, args) = getopt.gnu_getopt(args, "n", ['dry'])
660
dry = '-n' in opts or '--dry' in opts
663
print "Dry run (no actions will be executed\n"
665
if not dry and os.geteuid() != 0:
666
print >>sys.stderr, "Must be root to run install"
667
print >>sys.stderr, "(I need to chown some files)."
670
# Update the template jail directory in case it hasn't been installed
672
action_copytree('jail', os.path.join(jail_base, 'template'), dry)
674
# Re-link all the files in all students jails.
675
for dir in os.listdir(jail_base):
676
if dir == 'template': continue
677
# First back up the student's home directory
678
temp_home = os.tmpnam()
679
action_rename(os.path.join(jail_base, dir, 'home'), temp_home, dry)
680
# Delete the student's jail and relink the jail files
681
action_linktree(os.path.join(jail_base, 'template'),
682
os.path.join(jail_base, dir), dry)
683
# Restore the student's home directory
684
action_rename(temp_home, os.path.join(jail_base, dir, 'home'), dry)
685
# Set up the user's home directory just in case they don't have a
686
# directory for this yet
687
action_mkdir(os.path.join(jail_base, dir, 'home', dir), dry)
691
# The actions call Python os functions but print actions and handle dryness.
692
# May still throw os exceptions if errors occur.
695
"""Represents an error when running a program (nonzero return)."""
696
def __init__(self, prog, retcode):
698
self.retcode = retcode
700
return str(self.prog) + " returned " + repr(self.retcode)
702
def action_runprog(prog, args, dry):
703
"""Runs a unix program. Searches in $PATH. Synchronous (waits for the
704
program to return). Runs in the current environment. First prints the
705
action as a "bash" line.
707
Throws a RunError with a retcode of the return value of the program,
708
if the program did not return 0.
710
prog: String. Name of the program. (No path required, if in $PATH).
711
args: [String]. Arguments to the program.
712
dry: Bool. If True, prints but does not execute.
714
print prog, string.join(args, ' ')
716
ret = os.spawnvp(os.P_WAIT, prog, args)
718
raise RunError(prog, ret)
720
def action_rename(src, dst, dry):
721
"""Calls rename. Deletes the target if it already exists."""
722
if os.access(dst, os.F_OK):
725
shutil.rmtree(dst, True)
726
print "mv ", src, dst
730
except OSError, (err, msg):
731
if err != errno.EEXIST:
734
def action_mkdir(path, dry):
735
"""Calls mkdir. Silently ignored if the directory already exists.
736
Creates all parent directories as necessary."""
737
print "mkdir -p", path
741
except OSError, (err, msg):
742
if err != errno.EEXIST:
745
def action_copytree(src, dst, dry):
746
"""Copies an entire directory tree. Symlinks are seen as normal files and
747
copies of the entire file (not the link) are made. Creates all parent
748
directories as necessary.
750
See shutil.copytree."""
751
if os.access(dst, os.F_OK):
754
shutil.rmtree(dst, True)
755
print "cp -r", src, dst
757
shutil.copytree(src, dst, True)
759
def action_linktree(src, dst, dry):
760
"""Hard-links an entire directory tree. Same as copytree but the created
761
files are hard-links not actual copies. Removes the existing destination.
763
if os.access(dst, os.F_OK):
766
shutil.rmtree(dst, True)
767
print "<cp with hardlinks> -r", src, dst
769
common.makeuser.linktree(src, dst)
771
def action_copylist(srclist, dst, dry):
772
"""Copies all files in a list to a new location. The files in the list
773
are read relative to the current directory, and their destinations are the
774
same paths relative to dst. Creates all parent directories as necessary.
776
for srcfile in srclist:
777
dstfile = os.path.join(dst, srcfile)
778
dstdir = os.path.split(dstfile)[0]
779
if not os.path.isdir(dstdir):
780
action_mkdir(dstdir, dry)
781
print "cp -f", srcfile, dstfile
784
shutil.copyfile(srcfile, dstfile)
785
shutil.copymode(srcfile, dstfile)
789
def action_copyfile(src, dst, dry):
790
"""Copies one file to a new location. Creates all parent directories
793
dstdir = os.path.split(dst)[0]
794
if not os.path.isdir(dstdir):
795
action_mkdir(dstdir, dry)
796
print "cp -f", src, dst
799
shutil.copyfile(src, dst)
800
shutil.copymode(src, dst)
804
def action_symlink(src, dst, dry):
805
"""Creates a symlink in a given location. Creates all parent directories
808
dstdir = os.path.split(dst)[0]
809
if not os.path.isdir(dstdir):
810
action_mkdir(dstdir, dry)
811
# Delete existing file
812
if os.path.exists(dst):
814
print "ln -fs", src, dst
818
def action_append(ivle_pth, ivle_www):
819
file = open(ivle_pth, 'a+')
820
file.write(ivle_www + '\n')
823
def action_chown_setuid(file, dry):
824
"""Chowns a file to root, and sets the setuid bit on the file.
825
Calling this function requires the euid to be root.
826
The actual mode of path is set to: rws--s--s
828
print "chown root:root", file
831
print "chmod a+xs", file
832
print "chmod u+rw", file
834
os.chmod(file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
835
| stat.S_ISUID | stat.S_IRUSR | stat.S_IWUSR)
837
def action_chmod_x(file, dry):
838
"""Chmod +xs a file (sets execute permission)."""
839
print "chmod u+rwx", file
841
os.chmod(file, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
843
def query_user(default, prompt):
844
"""Prompts the user for a string, which is read from a line of stdin.
845
Exits silently if EOF is encountered. Returns the string, with spaces
846
removed from the beginning and end.
848
Returns default if a 0-length line (after spaces removed) was read.
850
sys.stdout.write('%s\n (default: "%s")\n>' % (prompt, default))
852
val = sys.stdin.readline()
853
except KeyboardInterrupt:
855
sys.stdout.write("\n")
857
sys.stdout.write("\n")
859
if val == '': sys.exit(1)
860
# If empty line, return default
862
if val == '': return default
865
def filter_mutate(function, list):
866
"""Like built-in filter, but mutates the given list instead of returning a
867
new one. Returns None."""
870
# Delete elements which do not match
871
if not function(list[i]):
88
875
if __name__ == "__main__":
added
removed
setup.py
