2
# IVLE - Informatics Virtual Learning Environment
3
# Copyright (C) 2007-2009 The University of Melbourne
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19
# Program: Fetch Submissions
22
# Script to retrieve all submissions for a particular project.
23
# Requires root to run.
25
from __future__ import with_statement
40
print >>sys.stderr, "Must run %s as root." % os.path.basename(sys.argv[0])
47
from ivle.database import Project, ProjectSet, Offering, Subject
50
locale.setlocale(locale.LC_CTYPE, "en_US.UTF-8")
52
# Is Python version 2.6 or higher?
53
PYTHON26 = map(int, sys.version[:3].split('.')) >= [2, 6]
55
def fetch_submission(submission, target, svnclient, config, zip=False,
56
txt=False, verbose=False):
57
"""Fetch a submission from a user's repository, and dump it in a given
59
@param submission: Submission object, detailing the submission.
60
@param target: Target directory for the project (will create a
61
subdirectory for each submission).
62
@param svnclient: pysvn.Client object.
63
@param config: Config object.
64
@param zip: If True, zips up the submission.
65
@param txt: If True, writes an extra text file with metadata about the
67
@param verbose: If True, writes the name of each submission to stdout.
69
# submission_name is the name of the user or group who owns the repo
70
submission_name = submission.assessed.principal.short_name
71
# target_final is the directory to place the files in
72
target_final = os.path.join(target,
73
submission.assessed.principal.short_name)
74
target_final = target_final.encode('utf-8')
75
if os.path.exists(target_final):
76
# Remove the existing file or directory before re-checking out
77
if os.path.isdir(target_final):
78
ivle.util.safe_rmtree(target_final)
80
os.remove(target_final)
81
url = get_repo_url(submission, config)
82
revision = pysvn.Revision(pysvn.opt_revision_kind.number,
84
svnclient.export(url, target_final, force=True,
85
revision=revision, recurse=True)
88
# info_final is the directory to place the metadata info files in
89
info_final = target_final + ".txt"
90
write_submission_info(info_final, submission)
92
# If required, zip up the directory
94
make_zip(target_final, target_final + ".zip")
95
# Remove the target tree
96
if os.path.isdir(target_final):
97
ivle.util.safe_rmtree(target_final)
99
os.remove(target_final)
102
print "Exported submission by: %s (%s)" % (
103
submission.assessed.principal.short_name,
104
submission.assessed.principal.display_name)
106
def get_repo_url(submission, config):
107
"""Gets a local (file:) URL to the repository for a given submission.
108
This will consult submission.path to find the path within the repository
110
@param submission: Submission object, detailing the submission.
111
@param config: Config object.
113
# NOTE: This code is mostly copied from services/usrmgt-server
114
if submission.assessed.is_group:
115
# The offering this group is in
116
offering = submission.assessed.project.project_set.offering
117
groupname = submission.assessed.principal.short_name
118
# The name of the repository directory within 'groups' is
119
# SUBJECT_YEAR_SEMESTER_GROUP
120
namespace = "_".join([offering.subject.short_name,
121
offering.semester.year, offering.semester.semester, groupname])
122
repo_path = os.path.join(config['paths']['svn']['repo_path'],
125
# The name of the repository directory within 'users' is the username
126
username = submission.assessed.principal.short_name
127
repo_path = os.path.join(config['paths']['svn']['repo_path'],
130
path_in_repo = submission.path
131
# Change an absolute path into a relative one (to the top of SVN)
132
if path_in_repo[:1] == os.sep or path_in_repo[:1] == os.altsep:
133
path_in_repo = path_in_repo[1:]
135
# Attach "file://" to the front of the absolute path, to make it a URL
136
return "file://" + os.path.join(os.path.abspath(repo_path), path_in_repo)
138
def make_zip(source, dest):
139
"""Zip up a directory tree or file. The ZIP file will always contain just
140
a single directory or file at the top level (it will not be a ZIP bomb).
141
XXX In Python 2.5 and earlier, this does NOT create empty directories
142
(it's not possible with the Python2.5 version of zipfile).
143
@param source: Path to a directory or file to zip up.
144
@param dest: Path to a zip file to create.
146
# NOTE: This code is mostly copied from ivle.zip (but very different)
147
zip = zipfile.ZipFile(dest, 'w')
149
# Write the source file/directory itself
150
# (If this is a directory it will NOT recurse)
151
if PYTHON26 or not os.path.isdir(source):
152
# Python < 2.6 errors if you add a directory
153
# (This means you can't add an empty directory)
154
zip.write(source, os.path.basename(source))
156
if os.path.isdir(source):
157
# All paths within the zip file are relative to relativeto
158
relativeto = os.path.dirname(source)
159
# Write the top-level directory
160
# Walk the directory tree
162
raise OSError("Could not access a file (zipping)")
163
for (dirpath, dirnames, filenames) in \
164
os.walk(source, onerror=error):
165
arc_dirpath = ivle.util.relpath(dirpath, relativeto)
166
# Python < 2.6 errors if you add a directory
167
# (This means you can't add an empty directory)
169
filenames = dirnames + filenames
170
for filename in filenames:
171
zip.write(os.path.join(dirpath, filename),
172
os.path.join(arc_dirpath, filename))
175
# XXX Write this _didModify attribute of zip, to trick it into writing
176
# footer bytes even if there are no files in the archive (otherwise it
177
# writes a 0-byte archive which is invalid).
178
# Note that in Python2.6 we avoid this by always writing the top-level
179
# file or directory, at least.
180
zip._didModify = True
183
def write_submission_info(filename, submission):
184
"""Write human-readable meta-data about a submission to a file.
185
@param filename: Filename to write to.
186
@param submission: Submission object.
188
with codecs.open(filename, 'w', 'utf-8') as f:
189
if submission.assessed.is_group:
191
print >>f, "Group: %s (%s)" % (
192
submission.assessed.principal.short_name,
193
submission.assessed.principal.display_name)
196
# Only show the two fields if they are different (only in rare
198
if submission.assessed.principal != submission.submitter:
199
print >>f, "Author: %s (%s)" % (
200
submission.assessed.principal.short_name,
201
submission.assessed.principal.display_name)
202
print >>f, "Submitter: %s (%s)" % (
203
submission.submitter.short_name,
204
submission.submitter.display_name)
205
print >>f, "Date: %s" % (
206
submission.date_submitted.strftime("%Y-%m-%d %H:%M:%S"))
207
print >>f, "SVN Revision: %s" % submission.revision
208
print >>f, "SVN Path: %s" % submission.path
215
usage = """usage: %prog [options] subject projname
217
Retrieves all submissions for a given project. Places each submission in
218
its own directory, in a subdirectory of '.'. Any errors are reported to
219
stderr (otherwise is silent).
220
subject/projname is the subject/project's short name.
224
parser = optparse.OptionParser(usage)
225
parser.add_option("-s", "--semester",
226
action="store", dest="semester", metavar="YEAR/SEMESTER",
227
help="Semester of the subject's offering (eg. 2009/1). "
228
"Defaults to the currently active semester.",
230
parser.add_option("-d", "--dest",
231
action="store", dest="dest", metavar="PATH",
232
help="Destination directory (default to '.', creates a subdirectory, "
233
"so will not pollute PATH).",
235
parser.add_option("-z", "--zip",
236
action="store_true", dest="zip",
237
help="Store each submission in a Zip file.",
239
parser.add_option("-v", "--verbose",
240
action="store_true", dest="verbose",
241
help="Print out the name of each submission as it is extracted.",
243
parser.add_option("--no-txt",
244
action="store_false", dest="txt",
245
help="Disable writing a text file with data about each submission.",
247
(options, args) = parser.parse_args(argv[1:])
253
subject_name = unicode(args[0])
254
project_name = unicode(args[1])
256
if options.semester is None:
257
year, semester = None, None
260
year, semester = options.semester.split('/')
261
if len(year) == 0 or len(semester) == 0:
264
parser.error('Invalid semester (must have form "year/semester")')
266
svnclient = pysvn.Client()
267
config = ivle.config.Config(plugins=False)
268
store = ivle.database.get_store(config)
270
# Get the subject from the DB
271
subject = store.find(Subject,
272
Subject.short_name == subject_name).one()
274
print >>sys.stderr, "No subject with short name '%s'" % subject_name
277
# Get the offering from the DB
279
# None specified - get the current offering from the DB
280
offerings = list(subject.active_offerings())
281
if len(offerings) == 0:
282
print >>sys.stderr, ("No active offering for subject '%s'"
285
elif len(offerings) > 1:
286
print >>sys.stderr, ("Multiple active offerings for subject '%s':"
288
print >>sys.stderr, "Please use one of:"
289
for offering in offerings:
290
print >>sys.stderr, (" --semester=%s/%s"
291
% (offering.semester.year, offering.semester.semester))
294
offering = offerings[0]
296
# Get the offering for the specified semester
297
offering = subject.offering_for_semester(year, semester)
299
print >>sys.stderr, (
300
"No offering for subject '%s' in semester %s/%s"
301
% (subject_name, year, semester))
304
# Get the project from the DB
305
project = store.find(Project,
306
Project.project_set_id == ProjectSet.id,
307
ProjectSet.offering == offering,
308
Project.short_name == project_name).one()
310
print >>sys.stderr, "No project with short name '%s'" % project_name
313
# Target directory is DEST/subject/year/semester/project
314
target_dir = os.path.join(options.dest, subject_name,
315
offering.semester.year, offering.semester.semester, project_name)
316
if not os.path.exists(target_dir):
317
os.makedirs(target_dir)
319
for submission in project.latest_submissions:
321
fetch_submission(submission, target_dir, svnclient, config,
322
zip=options.zip, txt=options.txt,
323
verbose=options.verbose)
325
# Catch all exceptions (to ensure if one student has a problem, it
326
# is reported, and we can continue)
327
print >>sys.stderr, "ERROR on submission for %s:" % (
328
submission.assessed.principal.display_name)
329
traceback.print_exc()
331
if __name__ == "__main__":
332
sys.exit(main(sys.argv))
added
removed
bin/ivle-fetchsubmissions
