418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
1 |
#!/usr/bin/python
|
2 |
||
3 |
# usage:
|
|
603
by mattgiuca
Console now starts up in the user's home directory. |
4 |
# python-console <port> <magic> [<working-dir>]
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
5 |
|
6 |
import cjson |
|
7 |
import codeop |
|
8 |
import md5 |
|
9 |
import os |
|
10 |
import Queue |
|
11 |
import signal |
|
12 |
import socket |
|
13 |
import sys |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
14 |
import traceback |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
15 |
from threading import Thread |
526
by drtomc
python-console: trivial bugfix - missing import. |
16 |
from functools import partial |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
17 |
|
432
by drtomc
usrmgt: more work on this. Still some work to go. |
18 |
import common.chat |
19 |
||
598
by drtomc
console: send output back to the browser progressively. |
20 |
class Interrupt(Exception): |
628
by drtomc
console: Add output based interrupt. This allows users to interrupt long |
21 |
def __init__(self): |
22 |
Exception.__init__(self, "Interrupted!") |
|
598
by drtomc
console: send output back to the browser progressively. |
23 |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
24 |
class ExpiryTimer(object): |
25 |
def __init__(self, idle): |
|
26 |
self.idle = idle |
|
564
by drtomc
python-console: Fix the timeout code. |
27 |
signal.signal(signal.SIGALRM, partial(self.timeout)) |
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
28 |
|
29 |
def ping(self): |
|
30 |
signal.alarm(self.idle) |
|
31 |
||
32 |
def start(self, time): |
|
33 |
signal.alarm(time) |
|
34 |
||
35 |
def stop(self): |
|
36 |
self.ping() |
|
37 |
||
38 |
def timeout(self, signum, frame): |
|
39 |
sys.exit(1) |
|
40 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
41 |
class StdinFromWeb(object): |
42 |
def __init__(self, cmdQ, lineQ): |
|
43 |
self.cmdQ = cmdQ |
|
44 |
self.lineQ = lineQ |
|
45 |
||
46 |
def readline(self): |
|
47 |
self.cmdQ.put({"input":None}) |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
48 |
expiry.ping() |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
49 |
ln = self.lineQ.get() |
50 |
if 'chat' in ln: |
|
51 |
return ln['chat'] |
|
648
by drtomc
console: fix a trivial bug which caused it to loop if you got a syntax error in your first command. |
52 |
if 'interrupt' in ln: |
53 |
raise Interrupt() |
|
860
by dcoles
Console: A console server can now be asked to finish by sending a message with |
54 |
if 'terminate' in ln: |
55 |
sys.exit(0) |
|
56 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
57 |
|
598
by drtomc
console: send output back to the browser progressively. |
58 |
class StdoutToWeb(object): |
59 |
def __init__(self, cmdQ, lineQ): |
|
60 |
self.cmdQ = cmdQ |
|
61 |
self.lineQ = lineQ |
|
62 |
self.remainder = '' |
|
63 |
||
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
64 |
def _trim_incomplete_final(self, stuff): |
65 |
'''Trim an incomplete UTF-8 character from the end of a string.
|
|
66 |
Returns (trimmed_string, count_of_trimmed_bytes).
|
|
67 |
'''
|
|
68 |
tokill = incomplete_utf8_sequence(stuff) |
|
69 |
if tokill == 0: |
|
70 |
return (stuff, tokill) |
|
71 |
else: |
|
72 |
return (stuff[:-tokill], tokill) |
|
73 |
||
598
by drtomc
console: send output back to the browser progressively. |
74 |
def write(self, stuff): |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
75 |
# print will only give a non-file a unicode or str. There's no way
|
76 |
# to convince it to encode unicodes, so we have to do it ourselves.
|
|
77 |
# Yay for file special-cases (fileobject.c, PyFile_WriteObject).
|
|
78 |
# If somebody wants to write some other object to here, they do it
|
|
79 |
# at their own peril.
|
|
80 |
if isinstance(stuff, unicode): |
|
81 |
stuff = stuff.encode('utf-8') |
|
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
82 |
self.remainder = self.remainder + stuff |
83 |
||
84 |
# if there's less than 128 bytes, buffer
|
|
85 |
if len(self.remainder) < 128: |
|
641
by drtomc
console: slightly more aggressive output buffering - wait till we've at least |
86 |
return
|
87 |
||
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
88 |
# if there's lots, then send it in 1/2K blocks
|
89 |
while len(self.remainder) > 512: |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
90 |
# We send things as Unicode inside JSON, so we must only send
|
91 |
# complete UTF-8 characters.
|
|
92 |
(blk, count) = self._trim_incomplete_final(self.remainder[:512]) |
|
93 |
self.cmdQ.put({"output":blk.decode('utf-8', 'replace')}) |
|
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
94 |
expiry.ping() |
95 |
ln = self.lineQ.get() |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
96 |
self.remainder = self.remainder[512 - count:] |
660
by drtomc
console: buffering now tries to buffer enough, but not too much. |
97 |
|
98 |
# Finally, split the remainder up into lines, and ship all the
|
|
99 |
# completed lines off to the server.
|
|
100 |
lines = self.remainder.split("\n") |
|
598
by drtomc
console: send output back to the browser progressively. |
101 |
self.remainder = lines[-1] |
102 |
del lines[-1] |
|
103 |
||
104 |
if len(lines) > 0: |
|
599
by drtomc
console: improve end of line handling. |
105 |
lines.append('') |
106 |
text = "\n".join(lines) |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
107 |
self.cmdQ.put({"output":text.decode('utf-8', 'replace')}) |
598
by drtomc
console: send output back to the browser progressively. |
108 |
expiry.ping() |
109 |
ln = self.lineQ.get() |
|
110 |
if 'interrupt' in ln: |
|
111 |
raise Interrupt() |
|
112 |
||
113 |
def flush(self): |
|
599
by drtomc
console: improve end of line handling. |
114 |
if len(self.remainder) > 0: |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
115 |
(out, count) = self._trim_incomplete_final(self.remainder) |
116 |
self.cmdQ.put({"output":out.decode('utf-8', 'replace')}) |
|
599
by drtomc
console: improve end of line handling. |
117 |
expiry.ping() |
118 |
ln = self.lineQ.get() |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
119 |
# Leave incomplete characters in the buffer.
|
120 |
# Yes, this does mean that an incomplete character will be left
|
|
121 |
# off the end, but we discussed this and it was deemed best.
|
|
122 |
self.remainder = self.remainder[len(self.remainder)-count:] |
|
599
by drtomc
console: improve end of line handling. |
123 |
if 'interrupt' in ln: |
124 |
raise Interrupt() |
|
598
by drtomc
console: send output back to the browser progressively. |
125 |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
126 |
class WebIO(object): |
127 |
"""Provides a file like interface to the Web front end of the console.
|
|
128 |
You may print text to the console using write(), flush any buffered output
|
|
129 |
using flush(), or request text from the console using readline()"""
|
|
130 |
||
131 |
def __init__(self, cmdQ, lineQ): |
|
132 |
self.cmdQ = cmdQ |
|
133 |
self.lineQ = lineQ |
|
134 |
self.stdin = StdinFromWeb(self.cmdQ, self.lineQ) |
|
135 |
self.stdout = StdoutToWeb(self.cmdQ, self.lineQ) |
|
136 |
||
137 |
def write(self, stuff): |
|
138 |
self.stdout.write(stuff) |
|
139 |
||
140 |
def flush(self): |
|
141 |
self.stdout.flush() |
|
142 |
||
143 |
def readline(self): |
|
144 |
self.stdout.flush() |
|
145 |
return self.stdin.readline() |
|
146 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
147 |
class PythonRunner(Thread): |
148 |
def __init__(self, cmdQ, lineQ): |
|
149 |
self.cmdQ = cmdQ |
|
150 |
self.lineQ = lineQ |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
151 |
self.webio = WebIO(self.cmdQ, self.lineQ) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
152 |
Thread.__init__(self) |
153 |
||
154 |
def execCmd(self, cmd): |
|
155 |
try: |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
156 |
sys.stdin = self.webio |
157 |
sys.stdout = self.webio |
|
158 |
sys.stderr = self.webio |
|
869
by wagrant
python-console: Don't assume that our command will give a return value, |
159 |
# We don't expect a return value - 'single' symbol prints it.
|
160 |
eval(cmd, self.globs) |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
161 |
self.webio.flush() |
869
by wagrant
python-console: Don't assume that our command will give a return value, |
162 |
self.cmdQ.put({"okay": None}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
163 |
self.curr_cmd = '' |
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
164 |
except: |
165 |
tb = format_exc_start(start=1) |
|
750
by dcoles
Console: Flush current output before requesting input from Web |
166 |
self.webio.flush() |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
167 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
168 |
self.curr_cmd = '' |
169 |
||
170 |
def run(self): |
|
598
by drtomc
console: send output back to the browser progressively. |
171 |
self.globs = {} |
172 |
self.globs['__builtins__'] = globals()['__builtins__'] |
|
173 |
self.curr_cmd = '' |
|
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
174 |
|
175 |
while True: |
|
176 |
ln = self.lineQ.get() |
|
177 |
if 'chat' in ln: |
|
178 |
if self.curr_cmd == '': |
|
179 |
self.curr_cmd = ln['chat'] |
|
180 |
else: |
|
181 |
self.curr_cmd = self.curr_cmd + '\n' + ln['chat'] |
|
182 |
try: |
|
869
by wagrant
python-console: Don't assume that our command will give a return value, |
183 |
cmd = codeop.compile_command(self.curr_cmd, '<web session>') |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
184 |
if cmd is None: |
185 |
# The command was incomplete,
|
|
186 |
# so send back a None, so the
|
|
187 |
# client can print a '...'
|
|
188 |
self.cmdQ.put({"more":None}) |
|
189 |
else: |
|
190 |
self.execCmd(cmd) |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
191 |
except: |
192 |
tb = format_exc_start(start=3) |
|
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
193 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
750
by dcoles
Console: Flush current output before requesting input from Web |
194 |
self.webio.flush() |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
195 |
self.curr_cmd = '' |
196 |
if 'block' in ln: |
|
197 |
# throw away a partial command.
|
|
198 |
try: |
|
199 |
cmd = compile(ln['block'], "<web session>", 'exec'); |
|
200 |
self.execCmd(cmd) |
|
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
201 |
except: |
867
by wagrant
python-console: Fix traceback trimming when in block mode (worksheets). |
202 |
tb = format_exc_start(start=1) |
750
by dcoles
Console: Flush current output before requesting input from Web |
203 |
self.webio.flush() |
874
by wagrant
python-console: We are now a shiny UTF-8 console. Don't send anything |
204 |
self.cmdQ.put({"exc": ''.join(tb).decode('utf-8', 'replace')}) |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
205 |
self.curr_cmd = '' |
206 |
||
207 |
def daemonize(): |
|
208 |
if os.fork(): # launch child and... |
|
209 |
os._exit(0) # kill off parent |
|
210 |
os.setsid() |
|
211 |
if os.fork(): # launch child and... |
|
212 |
os._exit(0) # kill off parent again. |
|
213 |
os.umask(077) |
|
214 |
||
215 |
# The global 'magic' is the secret that the client and server share
|
|
216 |
# which is used to create and md5 digest to authenticate requests.
|
|
217 |
# It is assigned a real value at startup.
|
|
218 |
magic = '' |
|
219 |
||
220 |
cmdQ = Queue.Queue() |
|
221 |
lineQ = Queue.Queue() |
|
222 |
interpThread = PythonRunner(cmdQ, lineQ) |
|
223 |
||
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
224 |
# Default expiry time of 15 minutes
|
225 |
expiry = ExpiryTimer(15 * 60) |
|
226 |
||
432
by drtomc
usrmgt: more work on this. Still some work to go. |
227 |
def initializer(): |
228 |
interpThread.setDaemon(True) |
|
229 |
interpThread.start() |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
230 |
expiry.ping() |
432
by drtomc
usrmgt: more work on this. Still some work to go. |
231 |
|
232 |
def dispatch_msg(msg): |
|
522
by drtomc
Add quite a lot of stuff to get usrmgt happening. |
233 |
expiry.ping() |
432
by drtomc
usrmgt: more work on this. Still some work to go. |
234 |
lineQ.put({msg['cmd']:msg['text']}) |
235 |
return cmdQ.get() |
|
236 |
||
846
by wagrant
python-console: Print proper tracebacks on exceptions. They actually |
237 |
def format_exc_start(start=0): |
238 |
etype, value, tb = sys.exc_info() |
|
239 |
tbbits = traceback.extract_tb(tb)[start:] |
|
240 |
list = ['Traceback (most recent call last):\n'] |
|
241 |
list = list + traceback.format_list(tbbits) |
|
242 |
list = list + traceback.format_exception_only(etype, value) |
|
243 |
return ''.join(list) |
|
244 |
||
871
by mattgiuca
scripts/python-console: Added function incomplete_utf8_sequence, for use with |
245 |
def incomplete_utf8_sequence(byteseq): |
246 |
"""
|
|
247 |
str -> int
|
|
248 |
Given a UTF-8-encoded byte sequence (str), returns the number of bytes at
|
|
249 |
the end of the string which comprise an incomplete UTF-8 character
|
|
250 |
sequence.
|
|
251 |
||
252 |
If the string is empty or ends with a complete character OR INVALID
|
|
253 |
sequence, returns 0.
|
|
254 |
Otherwise, returns 1-3 indicating the number of bytes in the final
|
|
255 |
incomplete (but valid) character sequence.
|
|
256 |
||
257 |
Does not check any bytes before the final sequence for correctness.
|
|
258 |
||
259 |
>>> incomplete_utf8_sequence("")
|
|
260 |
0
|
|
261 |
>>> incomplete_utf8_sequence("xy")
|
|
262 |
0
|
|
263 |
>>> incomplete_utf8_sequence("xy\xc3\xbc")
|
|
264 |
0
|
|
265 |
>>> incomplete_utf8_sequence("\xc3")
|
|
266 |
1
|
|
267 |
>>> incomplete_utf8_sequence("\xbc\xc3")
|
|
268 |
1
|
|
269 |
>>> incomplete_utf8_sequence("xy\xbc\xc3")
|
|
270 |
1
|
|
271 |
>>> incomplete_utf8_sequence("xy\xe0\xa0")
|
|
272 |
2
|
|
273 |
>>> incomplete_utf8_sequence("xy\xf4")
|
|
274 |
1
|
|
275 |
>>> incomplete_utf8_sequence("xy\xf4\x8f")
|
|
276 |
2
|
|
277 |
>>> incomplete_utf8_sequence("xy\xf4\x8f\xa0")
|
|
278 |
3
|
|
279 |
"""
|
|
280 |
count = 0 |
|
281 |
expect = None |
|
282 |
for b in byteseq[::-1]: |
|
283 |
b = ord(b) |
|
284 |
count += 1 |
|
285 |
if b & 0x80 == 0x0: |
|
286 |
# 0xxxxxxx (single-byte character)
|
|
287 |
expect = 1 |
|
288 |
break
|
|
289 |
elif b & 0xc0 == 0x80: |
|
290 |
# 10xxxxxx (subsequent byte)
|
|
291 |
pass
|
|
292 |
elif b & 0xe0 == 0xc0: |
|
293 |
# 110xxxxx (start of 2-byte sequence)
|
|
294 |
expect = 2 |
|
295 |
break
|
|
296 |
elif b & 0xf0 == 0xe0: |
|
297 |
# 1110xxxx (start of 3-byte sequence)
|
|
298 |
expect = 3 |
|
299 |
break
|
|
300 |
elif b & 0xf8 == 0xf0: |
|
301 |
# 11110xxx (start of 4-byte sequence)
|
|
302 |
expect = 4 |
|
303 |
break
|
|
304 |
else: |
|
305 |
# Invalid byte
|
|
306 |
return 0 |
|
307 |
||
308 |
if count >= 4: |
|
309 |
# Seen too many "subsequent bytes", invalid
|
|
310 |
return 0 |
|
311 |
||
312 |
if expect is None: |
|
313 |
# We never saw a "first byte", invalid
|
|
314 |
return 0 |
|
315 |
||
316 |
# We now know expect and count
|
|
317 |
if count >= expect: |
|
318 |
# Complete, or we saw an invalid sequence
|
|
319 |
return 0 |
|
320 |
elif count < expect: |
|
321 |
# Incomplete
|
|
322 |
return count |
|
323 |
||
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
324 |
if __name__ == "__main__": |
325 |
port = int(sys.argv[1]) |
|
326 |
magic = sys.argv[2] |
|
603
by mattgiuca
Console now starts up in the user's home directory. |
327 |
if len(sys.argv) >= 4: |
328 |
# working_dir
|
|
329 |
os.chdir(sys.argv[3]) |
|
749
by dcoles
Console: Override the sys.path on the console process so it's search path follows the cwd of the console (like how the commandline console works in interactive mode). This means you can now import modules in the $HOME directory. Ideally this should be the cwd of the browser (or just $HOME if it's a stand alone console), but at present there is no facility to set the cwd of the console when initilized. (So, we'll just stick with standalone mode for the moment) |
330 |
# Make python's search path follow the cwd
|
331 |
sys.path[0] = '' |
|
662
by drtomc
console: set the environment variable HOME so matplotlib works in the console. |
332 |
os.environ['HOME'] = sys.argv[3] |
418
by mattgiuca
Renamed trunk/console to trunk/scripts. We are now able to put more scripts in |
333 |
|
432
by drtomc
usrmgt: more work on this. Still some work to go. |
334 |
common.chat.start_server(port, magic, True, dispatch_msg, initializer) |