1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
|
# IVLE
# Copyright (C) 2007-2008 The University of Melbourne
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
# Module: File Service / Listing
# Author: Matt Giuca
# Date: 10/1/2008
# Handles the return part of the 2-stage process of fileservice. This
# is both the directory listing, and the raw serving of non-directory files.
# File Service Format.
# If a non-directory file is requested, then the HTTP response body will be
# the verbatim bytes of that file (if the file is valid). The HTTP response
# headers will include the guessed content type of the file, and the header
# "X-IVLE-Return: File".
# Directory Listing Format.
# If the path requested is a directory, then the HTTP response body will be
# a valid JSON string describing the directory. The HTTP response headers
# will include the header "X-IVLE-Return: Dir".
#
# The JSON structure is as follows:
# * The top-level value is an object. It always contains the key "listing",
# whose value is the primary listing object. It may also contain a key
# "clipboard" which contains the clipboard object.
# * The value for "listing" is an object, with one member for each file in the
# directory, plus an additional member (key ".") for the directory itself.
# * Each member's key is the filename. Its value is an object, which has
# various members describing the file.
# The members of this object are as follows:
# * svnstatus: String. The svn status of the file. Either all files in a
# directory or no files have an svnstatus. String may take the values:
# - none - does not exist
# - unversioned - is not a versioned thing in this wc
# - normal - exists, but uninteresting.
# - added - is scheduled for addition
# - missing - under v.c., but is missing
# - deleted - scheduled for deletion
# - replaced - was deleted and then re-added
# - modified - text or props have been modified
# - merged - local mods received repos mods
# - conflicted - local mods received conflicting repos mods
# - ignored - a resource marked as ignored
# - obstructed - an unversioned resource is in the way of the versioned resource
# - external - an unversioned path populated by an svn:external property
# - incomplete - a directory doesn't contain a complete entries list
# (From pysvn)
# If svnstatus is "Missing" then the file has no other attributes.
# * published: Boolean. True if the file is published. (Marked by a
# .published file in the folder)
# * isdir: Boolean. True if the file is a directory. Always present unless
# svnstatus is "missing".
# * size: Number. Size of the file in bytes. Present for non-directory
# files.
# * type: String. Guessed mime type of the file. Present for non-directory
# files.
# * mtime: Number. Number of seconds elapsed since the epoch.
# The epoch is not defined (this is an arbitrary number used for sorting
# dates).
# * mtime_nice: String. Modification time of the file or directory. Always
# present unless svnstatus is "Missing". Human-friendly.
#
# Members are not guaranteed to be present - client code should always check
# for each member that it is present, and handle gracefully if a member is not
# present.
#
# The listing object is guaranteed to have a "." key. Use this key to
# determine whether the directory is under version control or not. If this
# member does NOT have a "svnstatus" key, or "svnstatus" is "unversioned",
# then the directory is not under revision control (and no other files will
# have "svnstatus" either).
#
# The top-level object MAY contain a "clipboard" key, which specifies the
# files copied to the clipboard. This can be used by the client to show the
# user what files will be pasted. At the very least, the client should take
# the presence or absence of a "clipboard" key as whether to grey out the
# "paste" button.
#
# The "clipboard" object has three members:
# * mode: String. Either "copy" or "cut".
# * base: String. Path relative to the user's root. The common path between
# the files.
# * files: Array of Strings. Each element is a filename relative to base.
# Base and files exactly correspond to the listing path and argument paths
# which were supplied during the last copy or cut request.
import os
import sys
import stat
import mimetypes
import urlparse
from cgi import parse_qs
import cjson
import pysvn
import ivle.svn
import ivle.date
from ivle import (util, studpath)
import ivle.conf.mimetypes
# Make a Subversion client object
svnclient = pysvn.Client()
# Whether or not to ignore dot files.
# TODO check settings!
ignore_dot_files = True
# Mime types
# application/json is the "best" content type but is not good for
# debugging because Firefox just tries to download it
mime_dirlisting = "text/plain"
#mime_dirlisting = "application/json"
def handle_return(req, return_contents):
"""
Perform the "return" part of the response.
This function returns the file or directory listing contained in
req.path. Sets the HTTP response code in req, writes additional headers,
and writes the HTTP response, if any.
If return_contents is True, and the path is a non-directory, returns the
contents of the file verbatim. If False, returns a directory listing
with a single file, ".", and info about the file.
If the path is a directory, return_contents is ignored.
"""
(user, jail, path) = studpath.url_to_jailpaths(req.path)
# FIXME: What to do about req.path == ""?
# Currently goes to 403 Forbidden.
urlpath = urlparse.urlparse(path)
path = urlpath[2]
json = None
if path is None:
req.status = req.HTTP_FORBIDDEN
req.headers_out['X-IVLE-Return-Error'] = 'Forbidden'
req.write("Forbidden")
elif not os.access(path, os.R_OK):
req.status = req.HTTP_NOT_FOUND
req.headers_out['X-IVLE-Return-Error'] = 'File not found'
req.write("File not found")
elif os.path.isdir(path):
# It's a directory. Return the directory listing.
req.content_type = mime_dirlisting
req.headers_out['X-IVLE-Return'] = 'Dir'
# TODO: Fix this dirty, dirty hack
newjson = get_dirlisting(req, svnclient, path)
if ("X-IVLE-Action-Error" in req.headers_out):
newjson["Error"] = req.headers_out["X-IVLE-Action-Error"]
req.write(cjson.encode(newjson))
elif return_contents:
# It's a file. Return the file contents.
# First get the mime type of this file
# (Note that importing ivle.util has already initialised mime types)
(type, _) = mimetypes.guess_type(path)
if type is None:
type = ivle.conf.mimetypes.default_mimetype
req.content_type = type
req.headers_out['X-IVLE-Return'] = 'File'
send_file(req, svnclient, path)
else:
# It's a file. Return a "fake directory listing" with just this file.
req.content_type = mime_dirlisting
req.headers_out['X-IVLE-Return'] = 'File'
req.write(cjson.encode(get_dirlisting(req, svnclient, path)))
def _get_revision_or_die(req, svnclient, path):
'''Looks for a revision specification in req's URL, returning the revision
specified. Returns None if there was no revision specified. Errors and
terminates the request if the specification was bad, or it doesn't exist
for the given path.
'''
# Work out the revisions from query
r_str = req.get_fieldstorage().getfirst("r")
revision = ivle.svn.revision_from_string(r_str)
# Was some revision specified AND (it didn't resolve OR it was nonexistent)
if r_str and not (revision and
ivle.svn.revision_exists(svnclient, path, revision)):
req.status = req.HTTP_NOT_FOUND
req.headers_out['X-IVLE-Return-Error'] = 'Revision not found'
req.ensure_headers_written()
req.write('Revision not found')
req.flush()
sys.exit()
return revision
def send_file(req, svnclient, path):
revision = _get_revision_or_die(req, svnclient, path)
if revision:
req.write(svnclient.cat(path, revision=revision))
else:
req.sendfile(path)
def get_dirlisting(req, svnclient, path):
"""Given a local absolute path, creates a directory listing object
ready to be JSONized and sent to the client.
req: Request object. Will not be mutated; just reads the session.
svnclient: Svn client object.
path: String. Absolute path on the local file system. Not checked,
must already be guaranteed safe. May be a file or a directory.
"""
revision = _get_revision_or_die(req, svnclient, path)
# Start by trying to do an SVN status, so we can report file version
# status
listing = {}
try:
if revision:
ls_list = svnclient.list(path, revision=revision, recurse=False)
for ls in ls_list:
filename, attrs = PysvnList_to_fileinfo(path, ls)
listing[filename.decode('utf-8')] = attrs
else:
status_list = svnclient.status(path, recurse=False, get_all=True,
update=False)
for status in status_list:
filename, attrs = PysvnStatus_to_fileinfo(path, status)
listing[filename.decode('utf-8')] = attrs
except pysvn.ClientError:
# Presumably the directory is not under version control.
# Fallback to just an OS file listing.
try:
for filename in os.listdir(path):
listing[filename.decode('utf-8')] = file_to_fileinfo(path, filename)[1]
try:
svnclient.status(os.path.join(path, filename), recurse = False)
listing[filename.decode('utf-8')]['svnstatus'] = 'normal'
except:
pass
except OSError:
# Non-directories will error - that's OK, we just want the "."
pass
# The subversion one includes "." while the OS one does not.
# Add "." to the output, so the caller can see we are
# unversioned.
listing["."] = file_to_fileinfo(path, "")[1]
if ignore_dot_files:
for fn in listing.keys():
if fn != "." and fn.startswith("."):
del listing[fn]
# Listing is a nested object inside the top-level JSON.
listing = {"listing" : listing}
if revision:
listing['revision'] = revision.number
# The other object is the clipboard, if present in the browser session.
# This can go straight from the session to JSON.
session = req.get_session()
if session and 'clipboard' in session:
# In CGI mode, we can't get our hands on the
# session (for the moment), so just leave it out.
listing['clipboard'] = session['clipboard']
return listing
def _fullpath_stat_fileinfo(fullpath):
file_stat = os.stat(fullpath)
return _stat_fileinfo(fullpath, file_stat)
def _stat_fileinfo(fullpath, file_stat):
d = {}
if stat.S_ISDIR(file_stat.st_mode):
d["isdir"] = True
d["type_nice"] = util.nice_filetype("/")
# Only directories can be published
d["published"] = studpath.published(fullpath)
else:
d["isdir"] = False
d["size"] = file_stat.st_size
(type, _) = mimetypes.guess_type(fullpath)
if type is None:
type = ivle.conf.mimetypes.default_mimetype
d["type"] = type
d["type_nice"] = util.nice_filetype(fullpath)
d["mtime"] = file_stat.st_mtime
d["mtime_nice"] = ivle.date.make_date_nice(file_stat.st_mtime)
d["mtime_short"] = ivle.date.make_date_nice_short(file_stat.st_mtime)
return d
def file_to_fileinfo(path, filename):
"""Given a filename (relative to a given path), gets all the info "ls"
needs to display about the filename. Returns pair mapping filename to
a dict containing a number of other fields."""
fullpath = path if filename in ('', '.') else os.path.join(path, filename)
return filename, _fullpath_stat_fileinfo(fullpath)
def PysvnStatus_to_fileinfo(path, status):
"""Given a PysvnStatus object, gets all the info "ls"
needs to display about the filename. Returns a pair mapping filename to
a dict containing a number of other fields."""
path = os.path.normcase(path)
fullpath = status.path
# If this is "." (the directory itself)
if path == os.path.normcase(fullpath):
# If this directory is unversioned, then we aren't
# looking at any interesting files, so throw
# an exception and default to normal OS-based listing.
if status.text_status == pysvn.wc_status_kind.unversioned:
raise pysvn.ClientError
# We actually want to return "." because we want its
# subversion status.
filename = "."
else:
filename = os.path.basename(fullpath)
text_status = status.text_status
d = {'svnstatus': str(text_status)}
try:
d.update(_fullpath_stat_fileinfo(fullpath))
except OSError:
# Here if, eg, the file is missing.
# Can't get any more information so just return d
pass
return filename, d
def PysvnList_to_fileinfo(path, list):
"""Given a List object from pysvn.Client.list, gets all the info "ls"
needs to display about the filename. Returns a pair mapping filename to
a dict containing a number of other fields."""
path = os.path.normcase(path)
pysvnlist = list[0]
fullpath = pysvnlist.path
# If this is "." (the directory itself)
if path == os.path.normcase(fullpath):
# If this directory is unversioned, then we aren't
# looking at any interesting files, so throw
# an exception and default to normal OS-based listing.
#if status.text_status == pysvn.wc_status_kind.unversioned:
# raise pysvn.ClientError
# We actually want to return "." because we want its
# subversion status.
filename = "."
else:
filename = os.path.basename(fullpath)
d = {'svnstatus': 'revision'} # A special status.
wrapped = ivle.svn.PysvnListStatWrapper(pysvnlist)
d.update(_stat_fileinfo(fullpath, wrapped))
return filename, d
|