2
# IVLE - Informatics Virtual Learning Environment
3
# Copyright (C) 2007-2009 The University of Melbourne
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19
# Program: Fetch Submissions
22
# Script to retrieve all submissions for a particular project.
23
# Requires root to run.
25
from __future__ import with_statement
39
print >>sys.stderr, "Must run %s as root." % os.path.basename(sys.argv[0])
46
from ivle.database import Project, ProjectSet, Offering, Subject
48
# Is Python version 2.6 or higher?
49
PYTHON26 = map(int, sys.version[:3].split('.')) >= [2, 6]
51
def fetch_submission(submission, target, svnclient, config, zip=False,
53
"""Fetch a submission from a user's repository, and dump it in a given
55
@param submission: Submission object, detailing the submission.
56
@param target: Target directory for the project (will create a
57
subdirectory for each submission).
58
@param svnclient: pysvn.Client object.
59
@param config: Config object.
60
@param zip: If True, zips up the submission.
61
@param txt: If True, writes an extra text file with metadata about the
64
# submission_name is the name of the user or group who owns the repo
65
submission_name = submission.assessed.principal.short_name
66
# target_final is the directory to place the files in
67
target_final = os.path.join(target,
68
submission.assessed.principal.short_name)
69
target_final = target_final.encode('utf-8')
70
if os.path.exists(target_final):
71
# Remove the existing directory before re-checking out
72
ivle.util.safe_rmtree(target_final)
73
url = get_repo_url(submission, config)
74
revision = pysvn.Revision(pysvn.opt_revision_kind.number,
76
svnclient.export(url, target_final, force=True,
77
revision=revision, recurse=True)
80
# info_final is the directory to place the metadata info files in
81
info_final = target_final + ".txt"
82
write_submission_info(info_final, submission)
84
# If required, zip up the directory
86
make_zip(target_final, target_final + ".zip")
87
# Remove the target tree
88
ivle.util.safe_rmtree(target_final)
90
def get_repo_url(submission, config):
91
"""Gets a local (file:) URL to the repository for a given submission.
92
This will consult submission.path to find the path within the repository
94
@param submission: Submission object, detailing the submission.
95
@param config: Config object.
97
# NOTE: This code is mostly copied from services/usrmgt-server
98
if submission.assessed.is_group:
99
# The offering this group is in
100
offering = submission.assessed.project.project_set.offering
101
groupname = submission.assessed.principal.short_name
102
# The name of the repository directory within 'groups' is
103
# SUBJECT_YEAR_SEMESTER_GROUP
104
namespace = "_".join([offering.subject.short_name,
105
offering.semester.year, offering.semester.semester, groupname])
106
repo_path = os.path.join(config['paths']['svn']['repo_path'],
109
# The name of the repository directory within 'users' is the username
110
username = submission.assessed.principal.short_name
111
repo_path = os.path.join(config['paths']['svn']['repo_path'],
114
path_in_repo = submission.path
115
# Change an absolute path into a relative one (to the top of SVN)
116
if path_in_repo[:1] == os.sep or path_in_repo[:1] == os.altsep:
117
path_in_repo = path_in_repo[1:]
119
# Attach "file://" to the front of the absolute path, to make it a URL
120
return "file://" + os.path.join(os.path.abspath(repo_path), path_in_repo)
122
def make_zip(source, dest):
123
"""Zip up a directory tree or file. The ZIP file will always contain just
124
a single directory or file at the top level (it will not be a ZIP bomb).
125
XXX In Python 2.5 and earlier, this does NOT create empty directories
126
(it's not possible with the Python2.5 version of zipfile).
127
@param source: Path to a directory or file to zip up.
128
@param dest: Path to a zip file to create.
130
# NOTE: This code is mostly copied from ivle.zip (but very different)
131
zip = zipfile.ZipFile(dest, 'w')
133
# Write the source file/directory itself
134
# (If this is a directory it will NOT recurse)
135
if PYTHON26 or not os.path.isdir(source):
136
# Python < 2.6 errors if you add a directory
137
# (This means you can't add an empty directory)
138
zip.write(source, os.path.basename(source))
140
if os.path.isdir(source):
141
# All paths within the zip file are relative to relativeto
142
relativeto = os.path.dirname(source)
143
# Write the top-level directory
144
# Walk the directory tree
146
raise OSError("Could not access a file (zipping)")
147
for (dirpath, dirnames, filenames) in \
148
os.walk(source, onerror=error):
149
arc_dirpath = ivle.util.relpath(dirpath, relativeto)
150
# Python < 2.6 errors if you add a directory
151
# (This means you can't add an empty directory)
153
filenames = dirnames + filenames
154
for filename in filenames:
155
zip.write(os.path.join(dirpath, filename),
156
os.path.join(arc_dirpath, filename))
159
# XXX Write this _didModify attribute of zip, to trick it into writing
160
# footer bytes even if there are no files in the archive (otherwise it
161
# writes a 0-byte archive which is invalid).
162
# Note that in Python2.6 we avoid this by always writing the top-level
163
# file or directory, at least.
164
zip._didModify = True
167
def write_submission_info(filename, submission):
168
"""Write human-readable meta-data about a submission to a file.
169
@param filename: Filename to write to.
170
@param submission: Submission object.
172
with codecs.open(filename, 'w', 'utf-8') as f:
173
if submission.assessed.is_group:
175
print >>f, "Group: %s (%s)" % (
176
submission.assessed.principal.short_name,
177
submission.assessed.principal.display_name)
180
# Only show the two fields if they are different (only in rare
182
if submission.assessed.principal != submission.submitter:
183
print >>f, "Author: %s (%s)" % (
184
submission.assessed.principal.short_name,
185
submission.assessed.principal.display_name)
186
print >>f, "Submitter: %s (%s)" % (
187
submission.submitter.short_name,
188
submission.submitter.display_name)
189
print >>f, "Date: %s" % (
190
submission.date_submitted.strftime("%Y-%m-%d %H:%M:%S"))
191
print >>f, "SVN Revision: %s" % submission.revision
192
print >>f, "SVN Path: %s" % submission.path
199
usage = """usage: %prog [options] subject projname
201
Retrieves all submissions for a given project. Places each submission in
202
its own directory, in a subdirectory of '.'. Any errors are reported to
203
stderr (otherwise is silent).
204
subject/projname is the subject/project's short name.
208
parser = optparse.OptionParser(usage)
209
parser.add_option("-s", "--semester",
210
action="store", dest="semester", metavar="YEAR/SEMESTER",
211
help="Semester of the subject's offering (eg. 2009/1). "
212
"Defaults to the currently active semester.",
214
parser.add_option("-d", "--dest",
215
action="store", dest="dest", metavar="PATH",
216
help="Destination directory (default to '.', creates a subdirectory, "
217
"so will not pollute PATH).",
219
parser.add_option("-z", "--zip",
220
action="store_true", dest="zip",
221
help="Store each submission in a Zip file.",
223
parser.add_option("--no-txt",
224
action="store_false", dest="txt",
225
help="Disable writing a text file with data about each submission.",
227
(options, args) = parser.parse_args(argv[1:])
233
subject_name = unicode(args[0])
234
project_name = unicode(args[1])
236
if options.semester is None:
237
year, semester = None, None
240
year, semester = options.semester.split('/')
241
if len(year) == 0 or len(semester) == 0:
244
parser.error('Invalid semester (must have form "year/semester")')
246
svnclient = pysvn.Client()
247
config = ivle.config.Config(plugins=False)
248
store = ivle.database.get_store(config)
250
# Get the subject from the DB
251
subject = store.find(Subject,
252
Subject.short_name == subject_name).one()
254
print >>sys.stderr, "No subject with short name '%s'" % subject_name
257
# Get the offering from the DB
259
# None specified - get the current offering from the DB
260
offerings = list(subject.active_offerings())
261
if len(offerings) == 0:
262
print >>sys.stderr, ("No active offering for subject '%s'"
265
elif len(offerings) > 1:
266
print >>sys.stderr, ("Multiple active offerings for subject '%s':"
268
print >>sys.stderr, "Please use one of:"
269
for offering in offerings:
270
print >>sys.stderr, (" --semester=%s/%s"
271
% (offering.semester.year, offering.semester.semester))
274
offering = offerings[0]
276
# Get the offering for the specified semester
277
offering = subject.offering_for_semester(year, semester)
279
print >>sys.stderr, (
280
"No offering for subject '%s' in semester %s/%s"
281
% (subject_name, year, semester))
284
# Get the project from the DB
285
project = store.find(Project,
286
Project.project_set_id == ProjectSet.id,
287
ProjectSet.offering == offering,
288
Project.short_name == project_name).one()
290
print >>sys.stderr, "No project with short name '%s'" % project_name
293
# Target directory is DEST/subject/year/semester/project
294
target_dir = os.path.join(options.dest, subject_name,
295
offering.semester.year, offering.semester.semester, project_name)
296
if not os.path.exists(target_dir):
297
os.makedirs(target_dir)
299
for submission in project.latest_submissions:
301
fetch_submission(submission, target_dir, svnclient, config,
302
zip=options.zip, txt=options.txt)
304
# Catch all exceptions (to ensure if one student has a problem, it
305
# is reported, and we can continue)
306
print >>sys.stderr, "ERROR on submission for %s:" % (
307
submission.assessed.principal.display_name)
308
traceback.print_exc()
310
if __name__ == "__main__":
311
sys.exit(main(sys.argv))
added
removed
bin/ivle-fetchsubmissions
