[pyrepl-checkins]
pyrepl/pyrepl commands.py,1.5,1.6 completing_reader.py,1.3,1.4
historical_reader.py,1.3,1.4 python_reader.py,1.5,1.6
reader.py,1.6,1.7 unix_console.py,1.6,1.7
mwh@codespeak.net
mwh@codespeak.net
Fri, 16 May 2003 15:18:03 +0200 (MEST)
Update of /cvs/pyrepl/pyrepl/pyrepl
In directory thoth.codespeak.net:/tmp/cvs-serv13884
Modified Files:
commands.py completing_reader.py historical_reader.py
python_reader.py reader.py unix_console.py
Log Message:
Unicode support! Well, after a fashion:
Currently python_reader makes no attempt to work out the correct
encoding and UnixConsole defaults to 'utf-8'. This is wrong, but it works
for me.
Reader.get_buffer returns a unicode string. Ilpo & I decided a while back
that this is the wrong thing to do.
Fixes for both should follow shortly.
Index: commands.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/commands.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** commands.py 17 Jan 2003 14:01:50 -0000 1.5
--- commands.py 16 May 2003 13:18:00 -0000 1.6
***************
*** 345,355 ****
self.reader.install_keymap()
! _qikeymap = tuple(
! [("\\%03o"%o, 'qIHelp') for o in range(256)])
class quoted_insert(Command):
kills_digit_arg = 0
def do(self):
! self.reader.console.install_keymap(_qikeymap)
class invalid_key(Command):
--- 345,358 ----
self.reader.install_keymap()
! class DefaultDict:
! def __contains__(self, whatever):
! return True
! def __getitem__(self, whatever):
! return 'qIHelp'
class quoted_insert(Command):
kills_digit_arg = 0
def do(self):
! self.reader.console.k = self.reader.console.keymap = DefaultDict()
class invalid_key(Command):
Index: completing_reader.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/completing_reader.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** completing_reader.py 17 Jan 2003 13:36:45 -0000 1.3
--- completing_reader.py 16 May 2003 13:18:01 -0000 1.4
***************
*** 211,219 ****
def get_stem(self):
st = self.syntax_table
b = self.buffer
p = self.pos - 1
! while p >= 0 and st[b[p]] == reader.SYNTAX_WORD:
p -= 1
! return ''.join(b[p+1:self.pos])
def get_completions(self, stem):
--- 211,220 ----
def get_stem(self):
st = self.syntax_table
+ SW = reader.SYNTAX_WORD
b = self.buffer
p = self.pos - 1
! while p >= 0 and st.get(b[p], SW) == SW:
p -= 1
! return u''.join(b[p+1:self.pos])
def get_completions(self, stem):
Index: historical_reader.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/historical_reader.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** historical_reader.py 17 Jan 2003 13:37:38 -0000 1.3
--- historical_reader.py 16 May 2003 13:18:01 -0000 1.4
***************
*** 69,73 ****
r = self.reader
if r.historyi != len(r.history):
! if ''.join(r.buffer) != r.history[r.historyi]:
r.buffer = list(r.history[r.historyi])
r.pos = len(r.buffer)
--- 69,73 ----
r = self.reader
if r.historyi != len(r.history):
! if r.get_buffer() != r.history[r.historyi]:
r.buffer = list(r.history[r.historyi])
r.pos = len(r.buffer)
***************
*** 219,223 ****
def select_item(self, i):
! self.transient_history[self.historyi] = ''.join(self.buffer)
buf = self.transient_history.get(i)
if buf is None:
--- 219,223 ----
def select_item(self, i):
! self.transient_history[self.historyi] = self.get_buffer()
buf = self.transient_history.get(i)
if buf is None:
***************
*** 232,236 ****
return self.transient_history.get(i, self.history[i])
else:
! return self.transient_history.get(i, ''.join(self.buffer))
R_prepare = R.prepare
--- 232,236 ----
return self.transient_history.get(i, self.history[i])
else:
! return self.transient_history.get(i, self.get_buffer())
R_prepare = R.prepare
***************
*** 264,268 ****
p = self.pos
i = self.historyi
! s = ''.join(self.buffer)
forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
while 1:
--- 264,268 ----
p = self.pos
i = self.historyi
! s = self.get_buffer()
forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
while 1:
***************
*** 292,296 ****
def finish(self):
self.R_finish()
! ret = ''.join(self.buffer)
for i, t in self.transient_history.items():
if i < len(self.history) and i != self.historyi:
--- 292,296 ----
def finish(self):
self.R_finish()
! ret = self.get_buffer()
for i, t in self.transient_history.items():
if i < len(self.history) and i != self.historyi:
Index: python_reader.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/python_reader.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** python_reader.py 16 May 2003 13:08:43 -0000 1.5
--- python_reader.py 16 May 2003 13:18:01 -0000 1.6
***************
*** 54,60 ****
def do(self):
r = self.reader
! text = ''.join(r.buffer)
try:
! code = r.compiler(text)
except (OverflowError, SyntaxError, ValueError):
self.finish = 1
--- 54,61 ----
def do(self):
r = self.reader
! text = r.get_buffer()
try:
! # ooh, look at the hack:
! code = r.compiler("#coding:utf-8\n"+text.encode('utf-8'))
except (OverflowError, SyntaxError, ValueError):
self.finish = 1
***************
*** 121,125 ****
def get_completions(self, stem):
! b = ''.join(self.buffer)
m = import_line_prog.match(b)
if m:
--- 122,126 ----
def get_completions(self, stem):
! b = self.get_buffer()
m = import_line_prog.match(b)
if m:
***************
*** 174,178 ****
def execute(self, text):
try:
! code = self.compile(text, '<input>', 'single')
if code.co_flags & 16:
self.compile = nested_compile
--- 175,181 ----
def execute(self, text):
try:
! # ooh, look at the hack:
! code = self.compile("# coding:utf8\n"+text.encode('utf-8'),
! '<input>', 'single')
if code.co_flags & 16:
self.compile = nested_compile
***************
*** 181,184 ****
--- 184,188 ----
else:
self.runcode(code)
+ sys.stdout.flush()
def interact(self):
***************
*** 222,226 ****
else:
if self.reader.finished:
! text = ''.join(self.reader.buffer)
self.restore()
if text:
--- 226,230 ----
else:
if self.reader.finished:
! text = self.reader.get_buffer()
self.restore()
if text:
Index: reader.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/reader.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** reader.py 17 Jan 2003 14:01:50 -0000 1.6
--- reader.py 16 May 2003 13:18:01 -0000 1.7
***************
*** 18,44 ****
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
! import types
from pyrepl import commands
from curses import ascii
def _make_unctrl_map():
! map = {}
! for i in range(256):
! c = chr(i)
! if c.isalpha() or ascii.isprint(c):
! map[c] = c
for i in range(32):
! c = chr(i)
! map[c] = '^' + chr(ord('A') + i - 1)
! map['\177'] = '^?'
for i in range(256):
! c = chr(i)
! if not map.has_key(c):
! map[c] = '\\%03o'%i
! return map
# disp_str proved to be a bottleneck for large inputs, so it's been
# rewritten in C; it's not required though.
try:
from _pyrepl_utils import disp_str, init_unctrl_map
--- 18,45 ----
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
! import types, unicodedata
from pyrepl import commands
from curses import ascii
def _make_unctrl_map():
! uc_map = {}
! for c in map(unichr, range(256)):
! if unicodedata.category(c)[0] <> 'C':
! uc_map[c] = c
for i in range(32):
! c = unichr(i)
! uc_map[c] = u'^' + unichr(ord('A') + i - 1)
! uc_map['\177'] = u'^?'
for i in range(256):
! c = unichr(i)
! if not uc_map.has_key(c):
! uc_map[c] = u'\\%03o'%i
! return uc_map
# disp_str proved to be a bottleneck for large inputs, so it's been
# rewritten in C; it's not required though.
try:
+ raise ImportError # currently it's borked by the unicode support
+
from _pyrepl_utils import disp_str, init_unctrl_map
***************
*** 48,52 ****
except ImportError:
def _my_unctrl(c, u=_make_unctrl_map()):
! return u[c]
def disp_str(buffer, join=''.join, uc=_my_unctrl):
--- 49,60 ----
except ImportError:
def _my_unctrl(c, u=_make_unctrl_map()):
! import unicodedata
! if c in u:
! return u[c]
! else:
! if unicodedata.category(c).startswith('C'):
! return '\u%04x'%(ord(c),)
! else:
! return c
def disp_str(buffer, join=''.join, uc=_my_unctrl):
***************
*** 77,86 ****
def make_default_syntax_table():
st = {}
! for c in map(chr, range(256)):
st[c] = SYNTAX_SYMBOL
! for c in [a for a in map(chr, range(256)) if a.isalpha()]:
st[c] = SYNTAX_WORD
! st['\n'] = st[' '] = SYNTAX_WHITESPACE
return st
--- 85,95 ----
def make_default_syntax_table():
+ # XXX perhaps should use some unicodedata here?
st = {}
! for c in map(unichr, range(256)):
st[c] = SYNTAX_SYMBOL
! for c in [a for a in map(unichr, range(256)) if a.isalpha()]:
st[c] = SYNTAX_WORD
! st[u'\n'] = st[u' '] = SYNTAX_WHITESPACE
return st
***************
*** 241,245 ****
especially efficient is certainly simple(r).
"""
! lines = ''.join(self.buffer).split("\n")
screen = []
screeninfo = []
--- 250,254 ----
especially efficient is certainly simple(r).
"""
! lines = self.get_buffer().split("\n")
screen = []
screeninfo = []
***************
*** 285,291 ****
b = self.buffer
p -= 1
! while p >= 0 and st[b[p]] <> SYNTAX_WORD:
p -= 1
! while p >= 0 and st[b[p]] == SYNTAX_WORD:
p -= 1
return p + 1
--- 294,300 ----
b = self.buffer
p -= 1
! while p >= 0 and st.get(b[p], SYNTAX_WORD) <> SYNTAX_WORD:
p -= 1
! while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
p -= 1
return p + 1
***************
*** 301,307 ****
st = self.syntax_table
b = self.buffer
! while p < len(b) and st[b[p]] <> SYNTAX_WORD:
p += 1
! while p < len(b) and st[b[p]] == SYNTAX_WORD:
p += 1
return p
--- 310,316 ----
st = self.syntax_table
b = self.buffer
! while p < len(b) and st.get(b[p], SYNTAX_WORD) <> SYNTAX_WORD:
p += 1
! while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
p += 1
return p
***************
*** 467,471 ****
self.console.beep()
return
-
cmd = cmd_class(self, event)
--- 476,479 ----
***************
*** 506,510 ****
while not self.finished:
self.handle1()
! return ''.join(self.buffer)
finally:
self.restore()
--- 514,518 ----
while not self.finished:
self.handle1()
! return self.get_buffer()
finally:
self.restore()
***************
*** 513,516 ****
--- 521,528 ----
self.keymap += ((spec, command),)
self.install_keymap()
+
+ def get_buffer(self):
+ """Return the current buffer as a unicode string."""
+ return u''.join(self.buffer)
def test():
Index: unix_console.py
===================================================================
RCS file: /cvs/pyrepl/pyrepl/pyrepl/unix_console.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** unix_console.py 16 May 2003 13:08:43 -0000 1.6
--- unix_console.py 16 May 2003 13:18:01 -0000 1.7
***************
*** 19,23 ****
import termios, curses, select, os, struct, types, errno, signal
! import re, time, sys
from fcntl import ioctl
from pyrepl.fancy_termios import tcgetattr, tcsetattr
--- 19,23 ----
import termios, curses, select, os, struct, types, errno, signal
! import re, time, sys, codecs, unicodedata
from fcntl import ioctl
from pyrepl.fancy_termios import tcgetattr, tcsetattr
***************
*** 112,121 ****
class UnixConsole(Console):
! def __init__(self, f_in=0, f_out=1, term=None):
if isinstance(f_in, int):
! self.input = os.fdopen(f_in)
self.input_fd = f_in
else:
! self.input = f_in
self.input_fd = f_in.fileno()
--- 112,126 ----
class UnixConsole(Console):
! def __init__(self, f_in=0, f_out=1, term=None, encoding=None):
! if encoding is None:
! encoding = 'utf-8'#sys.getdefaultencoding()
!
! self.encoding = encoding
!
if isinstance(f_in, int):
! self.input = codecs.getreader(encoding)(os.fdopen(f_in))
self.input_fd = f_in
else:
! self.input = codecs.getreader(encoding)(f_in)
self.input_fd = f_in.fileno()
***************
*** 192,195 ****
--- 197,204 ----
self.keymap = {}
+ def change_encoding(self, encoding):
+ self.input = codecs.getreader(encoding)(self.input.stream)
+ self.encoding = encoding
+
def install_keymap(self, new_keymap):
self.k = self.keymap = \
***************
*** 444,453 ****
break
self.__cmd_buf += c
! try:
k = self.k = self.k[c]
! except:
! self.__cmd_buf += self.getpending()
! k = "invalid-key"
! if type(k) is not types.DictType:
e = Event(k, self.__cmd_buf, self.__cmd_buf)
self.k = self.keymap
--- 453,467 ----
break
self.__cmd_buf += c
! if c not in self.k:
! for c2 in self.__cmd_buf:
! if unicodedata.category(c2).startswith('C'):
! self.__cmd_buf += self.getpending()
! k = "invalid-key"
! break
! else:
! k = "self-insert"
! else:
k = self.k = self.k[c]
! if not isinstance(k, dict):
e = Event(k, self.__cmd_buf, self.__cmd_buf)
self.k = self.keymap
***************
*** 502,506 ****
self.__tputs(text)
else:
! os.write(self.output_fd, text))
del self.__buffer[:]
--- 516,520 ----
self.__tputs(text)
else:
! os.write(self.output_fd, text.encode(self.encoding))
del self.__buffer[:]