2
# IVLE - Informatics Virtual Learning Environment
3
# Copyright (C) 2007-2009 The University of Melbourne
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19
# Program: Fetch Submissions
22
# Script to retrieve all submissions for a particular project.
23
# Requires root to run.
25
from __future__ import with_statement
39
print >>sys.stderr, "Must run %s as root." % os.path.basename(sys.argv[0])
46
from ivle.database import Project, ProjectSet, Offering, Subject
48
# Is Python version 2.6 or higher?
49
PYTHON26 = map(int, sys.version[:3].split('.')) >= [2, 6]
51
def fetch_submission(submission, target, svnclient, config, zip=False,
52
txt=False, verbose=False):
53
"""Fetch a submission from a user's repository, and dump it in a given
55
@param submission: Submission object, detailing the submission.
56
@param target: Target directory for the project (will create a
57
subdirectory for each submission).
58
@param svnclient: pysvn.Client object.
59
@param config: Config object.
60
@param zip: If True, zips up the submission.
61
@param txt: If True, writes an extra text file with metadata about the
63
@param verbose: If True, writes the name of each submission to stdout.
65
# submission_name is the name of the user or group who owns the repo
66
submission_name = submission.assessed.principal.short_name
67
# target_final is the directory to place the files in
68
target_final = os.path.join(target,
69
submission.assessed.principal.short_name)
70
target_final = target_final.encode('utf-8')
71
if os.path.exists(target_final):
72
# Remove the existing file or directory before re-checking out
73
if os.path.isdir(target_final):
74
ivle.util.safe_rmtree(target_final)
76
os.remove(target_final)
77
url = get_repo_url(submission, config)
78
revision = pysvn.Revision(pysvn.opt_revision_kind.number,
80
svnclient.export(url, target_final, force=True,
81
revision=revision, recurse=True)
84
# info_final is the directory to place the metadata info files in
85
info_final = target_final + ".txt"
86
write_submission_info(info_final, submission)
88
# If required, zip up the directory
90
make_zip(target_final, target_final + ".zip")
91
# Remove the target tree
92
if os.path.isdir(target_final):
93
ivle.util.safe_rmtree(target_final)
95
os.remove(target_final)
98
print "Exported submission by: %s (%s)" % (
99
submission.assessed.principal.short_name,
100
submission.assessed.principal.display_name)
102
def get_repo_url(submission, config):
103
"""Gets a local (file:) URL to the repository for a given submission.
104
This will consult submission.path to find the path within the repository
106
@param submission: Submission object, detailing the submission.
107
@param config: Config object.
109
# NOTE: This code is mostly copied from services/usrmgt-server
110
if submission.assessed.is_group:
111
# The offering this group is in
112
offering = submission.assessed.project.project_set.offering
113
groupname = submission.assessed.principal.short_name
114
# The name of the repository directory within 'groups' is
115
# SUBJECT_YEAR_SEMESTER_GROUP
116
namespace = "_".join([offering.subject.short_name,
117
offering.semester.year, offering.semester.semester, groupname])
118
repo_path = os.path.join(config['paths']['svn']['repo_path'],
121
# The name of the repository directory within 'users' is the username
122
username = submission.assessed.principal.short_name
123
repo_path = os.path.join(config['paths']['svn']['repo_path'],
126
path_in_repo = submission.path
127
# Change an absolute path into a relative one (to the top of SVN)
128
if path_in_repo[:1] == os.sep or path_in_repo[:1] == os.altsep:
129
path_in_repo = path_in_repo[1:]
131
# Attach "file://" to the front of the absolute path, to make it a URL
132
return "file://" + os.path.join(os.path.abspath(repo_path), path_in_repo)
134
def make_zip(source, dest):
135
"""Zip up a directory tree or file. The ZIP file will always contain just
136
a single directory or file at the top level (it will not be a ZIP bomb).
137
XXX In Python 2.5 and earlier, this does NOT create empty directories
138
(it's not possible with the Python2.5 version of zipfile).
139
@param source: Path to a directory or file to zip up.
140
@param dest: Path to a zip file to create.
142
# NOTE: This code is mostly copied from ivle.zip (but very different)
143
zip = zipfile.ZipFile(dest, 'w')
145
# Write the source file/directory itself
146
# (If this is a directory it will NOT recurse)
147
if PYTHON26 or not os.path.isdir(source):
148
# Python < 2.6 errors if you add a directory
149
# (This means you can't add an empty directory)
150
zip.write(source, os.path.basename(source))
152
if os.path.isdir(source):
153
# All paths within the zip file are relative to relativeto
154
relativeto = os.path.dirname(source)
155
# Write the top-level directory
156
# Walk the directory tree
158
raise OSError("Could not access a file (zipping)")
159
for (dirpath, dirnames, filenames) in \
160
os.walk(source, onerror=error):
161
arc_dirpath = ivle.util.relpath(dirpath, relativeto)
162
# Python < 2.6 errors if you add a directory
163
# (This means you can't add an empty directory)
165
filenames = dirnames + filenames
166
for filename in filenames:
167
zip.write(os.path.join(dirpath, filename),
168
os.path.join(arc_dirpath, filename))
171
# XXX Write this _didModify attribute of zip, to trick it into writing
172
# footer bytes even if there are no files in the archive (otherwise it
173
# writes a 0-byte archive which is invalid).
174
# Note that in Python2.6 we avoid this by always writing the top-level
175
# file or directory, at least.
176
zip._didModify = True
179
def write_submission_info(filename, submission):
180
"""Write human-readable meta-data about a submission to a file.
181
@param filename: Filename to write to.
182
@param submission: Submission object.
184
with codecs.open(filename, 'w', 'utf-8') as f:
185
if submission.assessed.is_group:
187
print >>f, "Group: %s (%s)" % (
188
submission.assessed.principal.short_name,
189
submission.assessed.principal.display_name)
192
# Only show the two fields if they are different (only in rare
194
if submission.assessed.principal != submission.submitter:
195
print >>f, "Author: %s (%s)" % (
196
submission.assessed.principal.short_name,
197
submission.assessed.principal.display_name)
198
print >>f, "Submitter: %s (%s)" % (
199
submission.submitter.short_name,
200
submission.submitter.display_name)
201
print >>f, "Date: %s" % (
202
submission.date_submitted.strftime("%Y-%m-%d %H:%M:%S"))
203
print >>f, "SVN Revision: %s" % submission.revision
204
print >>f, "SVN Path: %s" % submission.path
211
usage = """usage: %prog [options] subject projname
213
Retrieves all submissions for a given project. Places each submission in
214
its own directory, in a subdirectory of '.'. Any errors are reported to
215
stderr (otherwise is silent).
216
subject/projname is the subject/project's short name.
220
parser = optparse.OptionParser(usage)
221
parser.add_option("-s", "--semester",
222
action="store", dest="semester", metavar="YEAR/SEMESTER",
223
help="Semester of the subject's offering (eg. 2009/1). "
224
"Defaults to the currently active semester.",
226
parser.add_option("-d", "--dest",
227
action="store", dest="dest", metavar="PATH",
228
help="Destination directory (default to '.', creates a subdirectory, "
229
"so will not pollute PATH).",
231
parser.add_option("-z", "--zip",
232
action="store_true", dest="zip",
233
help="Store each submission in a Zip file.",
235
parser.add_option("-v", "--verbose",
236
action="store_true", dest="verbose",
237
help="Print out the name of each submission as it is extracted.",
239
parser.add_option("--no-txt",
240
action="store_false", dest="txt",
241
help="Disable writing a text file with data about each submission.",
243
(options, args) = parser.parse_args(argv[1:])
249
subject_name = unicode(args[0])
250
project_name = unicode(args[1])
252
if options.semester is None:
253
year, semester = None, None
256
year, semester = options.semester.split('/')
257
if len(year) == 0 or len(semester) == 0:
260
parser.error('Invalid semester (must have form "year/semester")')
262
svnclient = pysvn.Client()
263
config = ivle.config.Config(plugins=False)
264
store = ivle.database.get_store(config)
266
# Get the subject from the DB
267
subject = store.find(Subject,
268
Subject.short_name == subject_name).one()
270
print >>sys.stderr, "No subject with short name '%s'" % subject_name
273
# Get the offering from the DB
275
# None specified - get the current offering from the DB
276
offerings = list(subject.active_offerings())
277
if len(offerings) == 0:
278
print >>sys.stderr, ("No active offering for subject '%s'"
281
elif len(offerings) > 1:
282
print >>sys.stderr, ("Multiple active offerings for subject '%s':"
284
print >>sys.stderr, "Please use one of:"
285
for offering in offerings:
286
print >>sys.stderr, (" --semester=%s/%s"
287
% (offering.semester.year, offering.semester.semester))
290
offering = offerings[0]
292
# Get the offering for the specified semester
293
offering = subject.offering_for_semester(year, semester)
295
print >>sys.stderr, (
296
"No offering for subject '%s' in semester %s/%s"
297
% (subject_name, year, semester))
300
# Get the project from the DB
301
project = store.find(Project,
302
Project.project_set_id == ProjectSet.id,
303
ProjectSet.offering == offering,
304
Project.short_name == project_name).one()
306
print >>sys.stderr, "No project with short name '%s'" % project_name
309
# Target directory is DEST/subject/year/semester/project
310
target_dir = os.path.join(options.dest, subject_name,
311
offering.semester.year, offering.semester.semester, project_name)
312
if not os.path.exists(target_dir):
313
os.makedirs(target_dir)
315
for submission in project.latest_submissions:
317
fetch_submission(submission, target_dir, svnclient, config,
318
zip=options.zip, txt=options.txt,
319
verbose=options.verbose)
321
# Catch all exceptions (to ensure if one student has a problem, it
322
# is reported, and we can continue)
323
print >>sys.stderr, "ERROR on submission for %s:" % (
324
submission.assessed.principal.display_name)
325
traceback.print_exc()
327
if __name__ == "__main__":
328
sys.exit(main(sys.argv))
added
removed
bin/ivle-fetchsubmissions
