Context Navigation

Back to Ticket #149

Ticket #149: stringchain.patch.txt

File stringchain.patch.txt, 11.5 KB (added by Zooko, 14 years ago)

Line
1	diff -r 9adfe64ba138 foolscap/banana.py
2	--- a/foolscap/banana.py Sun Mar 14 18:44:52 2010 -0700
3	+++ b/foolscap/banana.py Thu Mar 25 02:53:09 2010 -0600
4	@@ -4,6 +4,8 @@ from twisted.internet import protocol, d
5	from twisted.internet import protocol, defer, reactor
6	from twisted.python.failure import Failure
7	from twisted.python import log
8	+
9	+from stringchain import stringchain
10
11	# make sure to import allslicers, so they all get registered. Even if the
12	# need for RootSlicer/etc goes away, do the import here anyway.
13	@@ -585,7 +587,7 @@ class Banana(protocol.Protocol):
14	# self.buffer with the inbound negotiation block.
15	self.negotiated = False
16	self.connectionAbandoned = False
17	- self.buffer = ''
18	+ self.buffer = stringchain.StringChain()
19
20	self.incomingVocabulary = {}
21	self.skipBytes = 0 # used to discard a single long token
22	@@ -701,26 +703,22 @@ class Banana(protocol.Protocol):
23	# buffer, assemble into tokens
24	# call self.receiveToken(token) with each
25	if self.skipBytes:
26	- if len(chunk) < self.skipBytes:
27	+ if len(chunk) <= self.skipBytes:
28	# skip the whole chunk
29	self.skipBytes -= len(chunk)
30	return
31	# skip part of the chunk, and stop skipping
32	chunk = chunk[self.skipBytes:]
33	self.skipBytes = 0
34	- buffer = self.buffer + chunk
35	+ self.buffer.append(chunk)
36
37	# Loop through the available input data, extracting one token per
38	# pass.
39
40	- while buffer:
41	- assert self.buffer != buffer, \
42	- ("Banana.handleData: no progress made: %s %s" %
43	- (repr(buffer),))
44	- self.buffer = buffer
45	+ while len(self.buffer):
46	+ first65 = self.buffer.popleft(65)
47	pos = 0
48	-
49	- for ch in buffer:
50	+ for ch in first65:
51	if ch >= HIGH_BIT_SET:
52	break
53	pos = pos + 1
54	@@ -729,19 +727,20 @@ class Banana(protocol.Protocol):
55	# all of it, to make it harder for someone to spam our
56	# logs.
57	raise BananaError("token prefix is limited to 64 bytes: "
58	- "but got %r" % (buffer[:200],))
59	+ "but got %r" % (self.buffer.popleft(200),))
60	else:
61	# we've run out of buffer without seeing the high bit, which
62	# means we're still waiting for header to finish
63	+ self.buffer.appendleft(first65)
64	return
65	assert pos <= 64
66
67	# At this point, the header and type byte have been received.
68	# The body may or may not be complete.
69
70	- typebyte = buffer[pos]
71	+ typebyte = first65[pos]
72	if pos:
73	- header = b1282int(buffer[:pos])
74	+ header = b1282int(first65[:pos])
75	else:
76	header = 0
77
78	@@ -811,7 +810,7 @@ class Banana(protocol.Protocol):
79	# them with extreme prejudice.
80	raise BananaError("oversized ERROR token")
81
82	- rest = buffer[pos+1:]
83	+ self.buffer.appendleft(first65[pos+1:])
84
85	# determine what kind of token it is. Each clause finishes in
86	# one of four ways:
87	@@ -830,7 +829,6 @@ class Banana(protocol.Protocol):
88	# being passed up to the current Unslicer
89
90	if typebyte == OPEN:
91	- buffer = rest
92	self.inboundOpenCount = header
93	if rejected:
94	if self.debugReceive:
95	@@ -853,7 +851,6 @@ class Banana(protocol.Protocol):
96	continue
97
98	elif typebyte == CLOSE:
99	- buffer = rest
100	count = header
101	if self.discardCount:
102	self.discardCount -= 1
103	@@ -865,7 +862,6 @@ class Banana(protocol.Protocol):
104	continue
105
106	elif typebyte == ABORT:
107	- buffer = rest
108	count = header
109	# TODO: this isn't really a Violation, but we need something
110	# to describe it. It does behave identically to what happens
111	@@ -892,28 +888,26 @@ class Banana(protocol.Protocol):
112
113	elif typebyte == ERROR:
114	strlen = header
115	- if len(rest) >= strlen:
116	+ if len(self.buffer) >= strlen:
117	# the whole string is available
118	- buffer = rest[strlen:]
119	- obj = rest[:strlen]
120	+ obj = self.buffer.popleft(strlen)
121	# handleError must drop the connection
122	self.handleError(obj)
123	return
124	else:
125	+ self.buffer.appendleft(first65[:pos+1])
126	return # there is more to come
127
128	elif typebyte == LIST:
129	raise BananaError("oldbanana peer detected, " +
130	"compatibility code not yet written")
131	#listStack.append((header, []))
132	- #buffer = rest
133
134	elif typebyte == STRING:
135	strlen = header
136	- if len(rest) >= strlen:
137	+ if len(self.buffer) >= strlen:
138	# the whole string is available
139	- buffer = rest[strlen:]
140	- obj = rest[:strlen]
141	+ obj = self.buffer.popleft(strlen)
142	# although it might be rejected
143	else:
144	# there is more to come
145	@@ -922,24 +916,23 @@ class Banana(protocol.Protocol):
146	# dropped
147	if self.debugReceive:
148	print "DROPPED some string bits"
149	- self.skipBytes = strlen - len(rest)
150	- self.buffer = ""
151	+ self.skipBytes = strlen - len(self.buffer)
152	+ self.buffer.clear()
153	+ else:
154	+ self.buffer.appendleft(first65[:pos+1])
155	return
156
157	elif typebyte == INT:
158	- buffer = rest
159	obj = int(header)
160	elif typebyte == NEG:
161	- buffer = rest
162	# -2**31 is too large for a positive int, so go through
163	# LongType first
164	obj = int(-long(header))
165	elif typebyte == LONGINT or typebyte == LONGNEG:
166	strlen = header
167	- if len(rest) >= strlen:
168	+ if len(self.buffer) >= strlen:
169	# the whole number is available
170	- buffer = rest[strlen:]
171	- obj = bytes_to_long(rest[:strlen])
172	+ obj = bytes_to_long(self.buffer.popleft(strlen))
173	if typebyte == LONGNEG:
174	obj = -obj
175	# although it might be rejected
176	@@ -948,33 +941,32 @@ class Banana(protocol.Protocol):
177	if rejected:
178	# drop all we have and note how much more should be
179	# dropped
180	- self.skipBytes = strlen - len(rest)
181	- self.buffer = ""
182	+ self.skipBytes = strlen - len(self.buffer)
183	+ self.buffer.clear()
184	+ else:
185	+ self.buffer.appendleft(first65[:pos+1])
186	return
187
188	elif typebyte == VOCAB:
189	- buffer = rest
190	obj = self.incomingVocabulary[header]
191	# TODO: bail if expanded string is too big
192	# this actually means doing self.checkToken(VOCAB, len(obj))
193	# but we have to make sure we handle the rejection properly
194
195	elif typebyte == FLOAT:
196	- if len(rest) >= 8:
197	- buffer = rest[8:]
198	- obj = struct.unpack("!d", rest[:8])[0]
199	+ if len(self.buffer) >= 8:
200	+ obj = struct.unpack("!d", self.buffer.popleft(8))[0]
201	else:
202	# this case is easier than STRING, because it is only 8
203	# bytes. We don't bother skipping anything.
204	+ self.buffer.appendleft(first65[:pos+1])
205	return
206
207	elif typebyte == PING:
208	- buffer = rest
209	self.sendPONG(header)
210	continue # otherwise ignored
211
212	elif typebyte == PONG:
213	- buffer = rest
214	continue # otherwise ignored
215
216	else:
217	@@ -997,7 +989,7 @@ class Banana(protocol.Protocol):
218
219	# while loop ends here
220
221	- self.buffer = ''
222	+ self.buffer.clear()
223
224
225	def handleOpen(self, openCount, objectCount, indexToken):
226	diff -r 9adfe64ba138 foolscap/test/bench_banana.py
227	--- /dev/null Thu Jan 01 00:00:00 1970 +0000
228	+++ b/foolscap/test/bench_banana.py Thu Mar 25 02:53:09 2010 -0600
229	@@ -0,0 +1,46 @@
230	+import StringIO
231	+from foolscap import storage
232	+
233	+class TestTransport(StringIO.StringIO):
234	+ disconnectReason = None
235	+ def loseConnection(self):
236	+ pass
237	+
238	+class B(object):
239	+ def setup_huge_string(self, N):
240	+ """ This is actually a test for acceptable performance, and it needs to
241	+ be made more explicit, perhaps by being moved into a separate
242	+ benchmarking suite instead of living in this test suite. """
243	+ self.banana = storage.StorageBanana()
244	+ self.banana.slicerClass = storage.UnsafeStorageRootSlicer
245	+ self.banana.unslicerClass = storage.UnsafeStorageRootUnslicer
246	+ self.banana.transport = TestTransport()
247	+ self.banana.connectionMade()
248	+ d = self.banana.send("a"*N)
249	+ d.addCallback(lambda res: self.banana.transport.getvalue())
250	+ def f(o):
251	+ self._encoded_huge_string = o
252	+ d.addCallback(f)
253	+ reactor.runUntilCurrent()
254	+
255	+ def bench_huge_string_decode(self, N):
256	+ """ This is actually a test for acceptable performance, and it needs to
257	+ be made more explicit, perhaps by being moved into a separate
258	+ benchmarking suite instead of living in this test suite. """
259	+ o = self._encoded_huge_string
260	+ # results = []
261	+ self.banana.prepare()
262	+ # d.addCallback(results.append)
263	+ CHOMP = 4096
264	+ for i in range(0, len(o), CHOMP):
265	+ self.banana.dataReceived(o[i:i+CHOMP])
266	+ # print results
267	+
268	+import sys
269	+from twisted.internet import reactor
270	+from pyutil import benchutil
271	+b = B()
272	+for N in 103, 104, 105, 106, 10**7:
273	+ print "%8d" % N,
274	+ sys.stdout.flush()
275	+ benchutil.rep_bench(b.bench_huge_string_decode, N, b.setup_huge_string)
276	diff -r 9adfe64ba138 foolscap/test/test_copyable.py
277	--- a/foolscap/test/test_copyable.py Sun Mar 14 18:44:52 2010 -0700
278	+++ b/foolscap/test/test_copyable.py Thu Mar 25 02:53:09 2010 -0600
279	@@ -143,8 +143,8 @@ class Copyable(TargetMixin, unittest.Tes
280	self.failUnlessEqual(f2.frames, [])
281	self.failUnlessEqual(f2.tb, None)
282	self.failUnlessEqual(f2.stack, [])
283	- self.failUnless(f2.traceback.find("raise RuntimeError") != -1,
284	- "no 'raise RuntimeError' in '%s'" % (f2.traceback,))
285	+ self.failUnless(f2.traceback.find("RuntimeError") != -1,
286	+ "no 'RuntimeError' in '%s'" % (f2.traceback,))
287
288	def testFailure2(self):
289	self.callingBroker.unsafeTracebacks = False

Download in other formats:

Original Format