418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
1 |
#!/usr/bin/python
|
2 |
||
3 |
# usage:
|
|
603
by mattgiuca
Console now starts up in the user's home directory. |
4 |
# python-console <port> <magic> [<working-dir>]
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
5 |
|
6 |
import cjson |
|
7 |
import codeop |
|
8 |
import md5 |
|
9 |
import os |
|
10 |
import Queue |
|
11 |
import signal |
|
12 |
import socket |
|
13 |
import sys |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
14 |
import traceback |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
15 |
from threading import Thread |
16 |
||
432
by drtomc
usrmgt: more work on this. Still some work to go. |
17 |
import common.chat |
18 |
||
598
by drtomc
console: send output back to the browser progressively. |
19 |
class Interrupt(Exception): |
628
by drtomc
console: Add output based interrupt. This allows users to interrupt long |
20 |
def __init__(self): |
21 |
Exception.__init__(self, "Interrupted!") |
|
598
by drtomc
console: send output back to the browser progressively. |
22 |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
23 |
class ExpiryTimer(object): |
24 |
def __init__(self, idle): |
|
25 |
self.idle = idle |
|
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
26 |
signal.signal(signal.SIGALRM, self.timeout) |
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
27 |
|
28 |
def ping(self): |
|
29 |
signal.alarm(self.idle) |
|
30 |
||
31 |
def start(self, time): |
|
32 |
signal.alarm(time) |
|
33 |
||
34 |
def stop(self): |
|
35 |
self.ping() |
|
36 |
||
37 |
def timeout(self, signum, frame): |
|
38 |
sys.exit(1) |
|
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
39 |
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
40 |
class StdinFromWeb(object): |
41 |
def __init__(self, cmdQ, lineQ): |
|
42 |
self.cmdQ = cmdQ |
|
43 |
self.lineQ = lineQ |
|
44 |
||
45 |
def readline(self): |
|
46 |
self.cmdQ.put({"input":None}) |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
47 |
expiry.ping() |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
48 |
ln = self.lineQ.get() |
49 |
if 'chat' in ln: |
|
50 |
return ln['chat'] |
|
648
by drtomc
console: fix a trivial bug which caused it to loop if you got a syntax error in your first command. |
51 |
if 'interrupt' in ln: |
52 |
raise Interrupt() |
|
860
by dcoles
Console: A console server can now be asked to finish by sending a message with |
53 |
if 'terminate' in ln: |
54 |
sys.exit(0) |
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
55 |
|
598
by drtomc
console: send output back to the browser progressively. |
56 |
class StdoutToWeb(object): |
57 |
def __init__(self, cmdQ, lineQ): |
|
58 |
self.cmdQ = cmdQ |
|
59 |
self.lineQ = lineQ |
|
60 |
self.remainder = '' |
|
61 |
||
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
62 |
def _trim_incomplete_final(self, stuff): |
63 |
'''Trim an incomplete UTF-8 character from the end of a string.
|
|
64 |
Returns (trimmed_string, count_of_trimmed_bytes).
|
|
65 |
'''
|
|
66 |
tokill = incomplete_utf8_sequence(stuff) |
|
67 |
if tokill == 0: |
|
68 |
return (stuff, tokill) |
|
69 |
else: |
|
70 |
return (stuff[:-tokill], tokill) |
|
71 |
||
598
by drtomc
console: send output back to the browser progressively. |
72 |
def write(self, stuff): |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
73 |
# print will only give a non-file a unicode or str. There's no way
|
74 |
# to convince it to encode unicodes, so we have to do it ourselves.
|
|
75 |
# Yay for file special-cases (fileobject.c, PyFile_WriteObject).
|
|
76 |
# If somebody wants to write some other object to here, they do it
|
|
77 |
# at their own peril.
|
|
78 |
if isinstance(stuff, unicode): |
|
79 |
stuff = stuff.encode('utf-8') |
|
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
80 |
self.remainder = self.remainder + stuff |
81 |
||
82 |
# if there's less than 128 bytes, buffer
|
|
83 |
if len(self.remainder) < 128: |
|
641
by drtomc
console: slightly more aggressive output buffering - wait till we've at least |
84 |
return
|
85 |
||
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
86 |
# if there's lots, then send it in 1/2K blocks
|
87 |
while len(self.remainder) > 512: |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
88 |
# We send things as Unicode inside JSON, so we must only send
|
89 |
# complete UTF-8 characters.
|
|
90 |
(blk, count) = self._trim_incomplete_final(self.remainder[:512]) |
|
91 |
self.cmdQ.put({"output":blk.decode('utf-8', 'replace')}) |
|
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
92 |
expiry.ping() |
93 |
ln = self.lineQ.get() |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
94 |
self.remainder = self.remainder[512 - count:] |
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
95 |
|
96 |
# Finally, split the remainder up into lines, and ship all the
|
|
97 |
# completed lines off to the server.
|
|
98 |
lines = self.remainder.split("\n") |
|
598
by drtomc
console: send output back to the browser progressively. |
99 |
self.remainder = lines[-1] |
100 |
del lines[-1] |
|
101 |
||
102 |
if len(lines) > 0: |
|
599
by drtomc
console: improve end of line handling. |
103 |
lines.append('') |
104 |
text = "\n".join(lines) |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
105 |
self.cmdQ.put({"output":text.decode('utf-8', 'replace')}) |
598
by drtomc
console: send output back to the browser progressively. |
106 |
expiry.ping() |
107 |
ln = self.lineQ.get() |
|
108 |
if 'interrupt' in ln: |
|
109 |
raise Interrupt() |
|
110 |
||
111 |
def flush(self): |
|
599
by drtomc
console: improve end of line handling. |
112 |
if len(self.remainder) > 0: |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
113 |
(out, count) = self._trim_incomplete_final(self.remainder) |
114 |
self.cmdQ.put({"output":out.decode('utf-8', 'replace')}) |
|
599
by drtomc
console: improve end of line handling. |
115 |
expiry.ping() |
116 |
ln = self.lineQ.get() |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
117 |
# Leave incomplete characters in the buffer.
|
118 |
# Yes, this does mean that an incomplete character will be left
|
|
119 |
# off the end, but we discussed this and it was deemed best.
|
|
120 |
self.remainder = self.remainder[len(self.remainder)-count:] |
|
599
by drtomc
console: improve end of line handling. |
121 |
if 'interrupt' in ln: |
122 |
raise Interrupt() |
|
598
by drtomc
console: send output back to the browser progressively. |
123 |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
124 |
class WebIO(object): |
125 |
"""Provides a file like interface to the Web front end of the console.
|
|
126 |
You may print text to the console using write(), flush any buffered output
|
|
127 |
using flush(), or request text from the console using readline()"""
|
|
128 |
||
129 |
def __init__(self, cmdQ, lineQ): |
|
130 |
self.cmdQ = cmdQ |
|
131 |
self.lineQ = lineQ |
|
132 |
self.stdin = StdinFromWeb(self.cmdQ, self.lineQ) |
|
133 |
self.stdout = StdoutToWeb(self.cmdQ, self.lineQ) |
|
134 |
||
135 |
def write(self, stuff): |
|
136 |
self.stdout.write(stuff) |
|
137 |
||
138 |
def flush(self): |
|
139 |
self.stdout.flush() |
|
140 |
||
141 |
def readline(self): |
|
142 |
self.stdout.flush() |
|
143 |
return self.stdin.readline() |
|
144 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
145 |
class PythonRunner(Thread): |
146 |
def __init__(self, cmdQ, lineQ): |
|
147 |
self.cmdQ = cmdQ |
|
148 |
self.lineQ = lineQ |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
149 |
self.webio = WebIO(self.cmdQ, self.lineQ) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
150 |
Thread.__init__(self) |
151 |
||
152 |
def execCmd(self, cmd): |
|
153 |
try: |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
154 |
sys.stdin = self.webio |
155 |
sys.stdout = self.webio |
|
156 |
sys.stderr = self.webio |
|
869
by wagrant
python-console: Don't assume that our command will give a return value, |
157 |
# We don't expect a return value - 'single' symbol prints it.
|
158 |
eval(cmd, self.globs) |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
159 |
self.webio.flush() |
869
by wagrant
python-console: Don't assume that our command will give a return value, |
160 |
self.cmdQ.put({"okay": None}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
161 |
self.curr_cmd = '' |
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
162 |
except: |
163 |
tb = format_exc_start(start=1) |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
164 |
self.webio.flush() |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
165 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
166 |
self.curr_cmd = '' |
167 |
||
168 |
def run(self): |
|
598
by drtomc
console: send output back to the browser progressively. |
169 |
self.globs = {} |
170 |
self.globs['__builtins__'] = globals()['__builtins__'] |
|
171 |
self.curr_cmd = '' |
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
172 |
|
173 |
while True: |
|
174 |
ln = self.lineQ.get() |
|
175 |
if 'chat' in ln: |
|
176 |
if self.curr_cmd == '': |
|
177 |
self.curr_cmd = ln['chat'] |
|
178 |
else: |
|
179 |
self.curr_cmd = self.curr_cmd + '\n' + ln['chat'] |
|
180 |
try: |
|
869
by wagrant
python-console: Don't assume that our command will give a return value, |
181 |
cmd = codeop.compile_command(self.curr_cmd, '<web session>') |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
182 |
if cmd is None: |
183 |
# The command was incomplete,
|
|
184 |
# so send back a None, so the
|
|
185 |
# client can print a '...'
|
|
186 |
self.cmdQ.put({"more":None}) |
|
187 |
else: |
|
188 |
self.execCmd(cmd) |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
189 |
except: |
190 |
tb = format_exc_start(start=3) |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
191 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
750
by dcoles
Console: Flush current output before requesting input from Web |
192 |
self.webio.flush() |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
193 |
self.curr_cmd = '' |
194 |
if 'block' in ln: |
|
195 |
# throw away a partial command.
|
|
196 |
try: |
|
197 |
cmd = compile(ln['block'], "<web session>", 'exec'); |
|
198 |
self.execCmd(cmd) |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
199 |
except: |
867
by wagrant
python-console: Fix traceback trimming when in block mode (worksheets). |
200 |
tb = format_exc_start(start=1) |
750
by dcoles
Console: Flush current output before requesting input from Web |
201 |
self.webio.flush() |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
202 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
203 |
self.curr_cmd = '' |
204 |
||
205 |
def daemonize(): |
|
206 |
if os.fork(): # launch child and... |
|
207 |
os._exit(0) # kill off parent |
|
208 |
os.setsid() |
|
209 |
if os.fork(): # launch child and... |
|
210 |
os._exit(0) # kill off parent again. |
|
211 |
os.umask(077) |
|
212 |
||
213 |
# The global 'magic' is the secret that the client and server share
|
|
214 |
# which is used to create and md5 digest to authenticate requests.
|
|
215 |
# It is assigned a real value at startup.
|
|
216 |
magic = '' |
|
217 |
||
218 |
cmdQ = Queue.Queue() |
|
219 |
lineQ = Queue.Queue() |
|
220 |
interpThread = PythonRunner(cmdQ, lineQ) |
|
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
221 |
terminate = None |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
222 |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
223 |
# Default expiry time of 15 minutes
|
224 |
expiry = ExpiryTimer(15 * 60) |
|
225 |
||
432
by drtomc
usrmgt: more work on this. Still some work to go. |
226 |
def initializer(): |
227 |
interpThread.setDaemon(True) |
|
228 |
interpThread.start() |
|
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
229 |
signal.signal(signal.SIGXCPU, sig_handler) |
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
230 |
expiry.ping() |
432
by drtomc
usrmgt: more work on this. Still some work to go. |
231 |
|
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
232 |
def sig_handler(signum, frame): |
233 |
"""Handles response from signals"""
|
|
234 |
global terminate |
|
235 |
if signum == signal.SIGXCPU: |
|
236 |
terminate = "CPU Time Limit Exceeded" |
|
237 |
||
432
by drtomc
usrmgt: more work on this. Still some work to go. |
238 |
def dispatch_msg(msg): |
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
239 |
expiry.ping() |
432
by drtomc
usrmgt: more work on this. Still some work to go. |
240 |
lineQ.put({msg['cmd']:msg['text']}) |
977
by dcoles
Console: Work on making console more responsive to signals (rather than just |
241 |
if terminate: |
242 |
raise common.chat.Terminate({"restart":terminate}) |
|
432
by drtomc
usrmgt: more work on this. Still some work to go. |
243 |
return cmdQ.get() |
244 |
||
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
245 |
def format_exc_start(start=0): |
246 |
etype, value, tb = sys.exc_info() |
|
247 |
tbbits = traceback.extract_tb(tb)[start:] |
|
248 |
list = ['Traceback (most recent call last):\n'] |
|
249 |
list = list + traceback.format_list(tbbits) |
|
250 |
list = list + traceback.format_exception_only(etype, value) |
|
251 |
return ''.join(list) |
|
252 |
||
871
by mattgiuca
scripts/python-console: Added function incomplete_utf8_sequence, for use with |
253 |
def incomplete_utf8_sequence(byteseq): |
254 |
"""
|
|
255 |
str -> int
|
|
256 |
Given a UTF-8-encoded byte sequence (str), returns the number of bytes at
|
|
257 |
the end of the string which comprise an incomplete UTF-8 character
|
|
258 |
sequence.
|
|
259 |
||
260 |
If the string is empty or ends with a complete character OR INVALID
|
|
261 |
sequence, returns 0.
|
|
262 |
Otherwise, returns 1-3 indicating the number of bytes in the final
|
|
263 |
incomplete (but valid) character sequence.
|
|
264 |
||
265 |
Does not check any bytes before the final sequence for correctness.
|
|
266 |
||
267 |
>>> incomplete_utf8_sequence("")
|
|
268 |
0
|
|
269 |
>>> incomplete_utf8_sequence("xy")
|
|
270 |
0
|
|
271 |
>>> incomplete_utf8_sequence("xy\xc3\xbc")
|
|
272 |
0
|
|
273 |
>>> incomplete_utf8_sequence("\xc3")
|
|
274 |
1
|
|
275 |
>>> incomplete_utf8_sequence("\xbc\xc3")
|
|
276 |
1
|
|
277 |
>>> incomplete_utf8_sequence("xy\xbc\xc3")
|
|
278 |
1
|
|
279 |
>>> incomplete_utf8_sequence("xy\xe0\xa0")
|
|
280 |
2
|
|
281 |
>>> incomplete_utf8_sequence("xy\xf4")
|
|
282 |
1
|
|
283 |
>>> incomplete_utf8_sequence("xy\xf4\x8f")
|
|
284 |
2
|
|
285 |
>>> incomplete_utf8_sequence("xy\xf4\x8f\xa0")
|
|
286 |
3
|
|
287 |
"""
|
|
288 |
count = 0 |
|
289 |
expect = None |
|
290 |
for b in byteseq[::-1]: |
|
291 |
b = ord(b) |
|
292 |
count += 1 |
|
293 |
if b & 0x80 == 0x0: |
|
294 |
# 0xxxxxxx (single-byte character)
|
|
295 |
expect = 1 |
|
296 |
break
|
|
297 |
elif b & 0xc0 == 0x80: |
|
298 |
# 10xxxxxx (subsequent byte)
|
|
299 |
pass
|
|
300 |
elif b & 0xe0 == 0xc0: |
|
301 |
# 110xxxxx (start of 2-byte sequence)
|
|
302 |
expect = 2 |
|
303 |
break
|
|
304 |
elif b & 0xf0 == 0xe0: |
|
305 |
# 1110xxxx (start of 3-byte sequence)
|
|
306 |
expect = 3 |
|
307 |
break
|
|
308 |
elif b & 0xf8 == 0xf0: |
|
309 |
# 11110xxx (start of 4-byte sequence)
|
|
310 |
expect = 4 |
|
311 |
break
|
|
312 |
else: |
|
313 |
# Invalid byte
|
|
314 |
return 0 |
|
315 |
||
316 |
if count >= 4: |
|
317 |
# Seen too many "subsequent bytes", invalid
|
|
318 |
return 0 |
|
319 |
||
320 |
if expect is None: |
|
321 |
# We never saw a "first byte", invalid
|
|
322 |
return 0 |
|
323 |
||
324 |
# We now know expect and count
|
|
325 |
if count >= expect: |
|
326 |
# Complete, or we saw an invalid sequence
|
|
327 |
return 0 |
|
328 |
elif count < expect: |
|
329 |
# Incomplete
|
|
330 |
return count |
|
331 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
332 |
if __name__ == "__main__": |
333 |
port = int(sys.argv[1]) |
|
334 |
magic = sys.argv[2] |
|
974
by dcoles
Console: Sanitise Environment varibles. We now only give PATH and HOME (and |
335 |
|
336 |
# Sanitise the Enviroment
|
|
337 |
os.environ = {} |
|
338 |
os.environ['PATH'] = '/usr/local/bin:/usr/bin:/bin' |
|
339 |
||
603
by mattgiuca
Console now starts up in the user's home directory. |
340 |
if len(sys.argv) >= 4: |
341 |
# working_dir
|
|
342 |
os.chdir(sys.argv[3]) |
|
662
by drtomc
console: set the environment variable HOME so matplotlib works in the console. |
343 |
os.environ['HOME'] = sys.argv[3] |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
344 |
|
974
by dcoles
Console: Sanitise Environment varibles. We now only give PATH and HOME (and |
345 |
# Make python's search path follow the cwd
|
346 |
sys.path[0] = '' |
|
347 |
||
432
by drtomc
usrmgt: more work on this. Still some work to go. |
348 |
common.chat.start_server(port, magic, True, dispatch_msg, initializer) |