diff -r 9adfe64ba138 foolscap/banana.py --- a/foolscap/banana.py Sun Mar 14 18:44:52 2010 -0700 +++ b/foolscap/banana.py Thu Mar 25 02:53:09 2010 -0600 @@ -4,6 +4,8 @@ from twisted.internet import protocol, d from twisted.internet import protocol, defer, reactor from twisted.python.failure import Failure from twisted.python import log + +from stringchain import stringchain # make sure to import allslicers, so they all get registered. Even if the # need for RootSlicer/etc goes away, do the import here anyway. @@ -585,7 +587,7 @@ class Banana(protocol.Protocol): # self.buffer with the inbound negotiation block. self.negotiated = False self.connectionAbandoned = False - self.buffer = '' + self.buffer = stringchain.StringChain() self.incomingVocabulary = {} self.skipBytes = 0 # used to discard a single long token @@ -701,26 +703,22 @@ class Banana(protocol.Protocol): # buffer, assemble into tokens # call self.receiveToken(token) with each if self.skipBytes: - if len(chunk) < self.skipBytes: + if len(chunk) <= self.skipBytes: # skip the whole chunk self.skipBytes -= len(chunk) return # skip part of the chunk, and stop skipping chunk = chunk[self.skipBytes:] self.skipBytes = 0 - buffer = self.buffer + chunk + self.buffer.append(chunk) # Loop through the available input data, extracting one token per # pass. - while buffer: - assert self.buffer != buffer, \ - ("Banana.handleData: no progress made: %s %s" % - (repr(buffer),)) - self.buffer = buffer + while len(self.buffer): + first65 = self.buffer.popleft(65) pos = 0 - - for ch in buffer: + for ch in first65: if ch >= HIGH_BIT_SET: break pos = pos + 1 @@ -729,19 +727,20 @@ class Banana(protocol.Protocol): # all of it, to make it harder for someone to spam our # logs. raise BananaError("token prefix is limited to 64 bytes: " - "but got %r" % (buffer[:200],)) + "but got %r" % (self.buffer.popleft(200),)) else: # we've run out of buffer without seeing the high bit, which # means we're still waiting for header to finish + self.buffer.appendleft(first65) return assert pos <= 64 # At this point, the header and type byte have been received. # The body may or may not be complete. - typebyte = buffer[pos] + typebyte = first65[pos] if pos: - header = b1282int(buffer[:pos]) + header = b1282int(first65[:pos]) else: header = 0 @@ -811,7 +810,7 @@ class Banana(protocol.Protocol): # them with extreme prejudice. raise BananaError("oversized ERROR token") - rest = buffer[pos+1:] + self.buffer.appendleft(first65[pos+1:]) # determine what kind of token it is. Each clause finishes in # one of four ways: @@ -830,7 +829,6 @@ class Banana(protocol.Protocol): # being passed up to the current Unslicer if typebyte == OPEN: - buffer = rest self.inboundOpenCount = header if rejected: if self.debugReceive: @@ -853,7 +851,6 @@ class Banana(protocol.Protocol): continue elif typebyte == CLOSE: - buffer = rest count = header if self.discardCount: self.discardCount -= 1 @@ -865,7 +862,6 @@ class Banana(protocol.Protocol): continue elif typebyte == ABORT: - buffer = rest count = header # TODO: this isn't really a Violation, but we need something # to describe it. It does behave identically to what happens @@ -892,28 +888,26 @@ class Banana(protocol.Protocol): elif typebyte == ERROR: strlen = header - if len(rest) >= strlen: + if len(self.buffer) >= strlen: # the whole string is available - buffer = rest[strlen:] - obj = rest[:strlen] + obj = self.buffer.popleft(strlen) # handleError must drop the connection self.handleError(obj) return else: + self.buffer.appendleft(first65[:pos+1]) return # there is more to come elif typebyte == LIST: raise BananaError("oldbanana peer detected, " + "compatibility code not yet written") #listStack.append((header, [])) - #buffer = rest elif typebyte == STRING: strlen = header - if len(rest) >= strlen: + if len(self.buffer) >= strlen: # the whole string is available - buffer = rest[strlen:] - obj = rest[:strlen] + obj = self.buffer.popleft(strlen) # although it might be rejected else: # there is more to come @@ -922,24 +916,23 @@ class Banana(protocol.Protocol): # dropped if self.debugReceive: print "DROPPED some string bits" - self.skipBytes = strlen - len(rest) - self.buffer = "" + self.skipBytes = strlen - len(self.buffer) + self.buffer.clear() + else: + self.buffer.appendleft(first65[:pos+1]) return elif typebyte == INT: - buffer = rest obj = int(header) elif typebyte == NEG: - buffer = rest # -2**31 is too large for a positive int, so go through # LongType first obj = int(-long(header)) elif typebyte == LONGINT or typebyte == LONGNEG: strlen = header - if len(rest) >= strlen: + if len(self.buffer) >= strlen: # the whole number is available - buffer = rest[strlen:] - obj = bytes_to_long(rest[:strlen]) + obj = bytes_to_long(self.buffer.popleft(strlen)) if typebyte == LONGNEG: obj = -obj # although it might be rejected @@ -948,33 +941,32 @@ class Banana(protocol.Protocol): if rejected: # drop all we have and note how much more should be # dropped - self.skipBytes = strlen - len(rest) - self.buffer = "" + self.skipBytes = strlen - len(self.buffer) + self.buffer.clear() + else: + self.buffer.appendleft(first65[:pos+1]) return elif typebyte == VOCAB: - buffer = rest obj = self.incomingVocabulary[header] # TODO: bail if expanded string is too big # this actually means doing self.checkToken(VOCAB, len(obj)) # but we have to make sure we handle the rejection properly elif typebyte == FLOAT: - if len(rest) >= 8: - buffer = rest[8:] - obj = struct.unpack("!d", rest[:8])[0] + if len(self.buffer) >= 8: + obj = struct.unpack("!d", self.buffer.popleft(8))[0] else: # this case is easier than STRING, because it is only 8 # bytes. We don't bother skipping anything. + self.buffer.appendleft(first65[:pos+1]) return elif typebyte == PING: - buffer = rest self.sendPONG(header) continue # otherwise ignored elif typebyte == PONG: - buffer = rest continue # otherwise ignored else: @@ -997,7 +989,7 @@ class Banana(protocol.Protocol): # while loop ends here - self.buffer = '' + self.buffer.clear() def handleOpen(self, openCount, objectCount, indexToken): diff -r 9adfe64ba138 foolscap/test/bench_banana.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/foolscap/test/bench_banana.py Thu Mar 25 02:53:09 2010 -0600 @@ -0,0 +1,46 @@ +import StringIO +from foolscap import storage + +class TestTransport(StringIO.StringIO): + disconnectReason = None + def loseConnection(self): + pass + +class B(object): + def setup_huge_string(self, N): + """ This is actually a test for acceptable performance, and it needs to + be made more explicit, perhaps by being moved into a separate + benchmarking suite instead of living in this test suite. """ + self.banana = storage.StorageBanana() + self.banana.slicerClass = storage.UnsafeStorageRootSlicer + self.banana.unslicerClass = storage.UnsafeStorageRootUnslicer + self.banana.transport = TestTransport() + self.banana.connectionMade() + d = self.banana.send("a"*N) + d.addCallback(lambda res: self.banana.transport.getvalue()) + def f(o): + self._encoded_huge_string = o + d.addCallback(f) + reactor.runUntilCurrent() + + def bench_huge_string_decode(self, N): + """ This is actually a test for acceptable performance, and it needs to + be made more explicit, perhaps by being moved into a separate + benchmarking suite instead of living in this test suite. """ + o = self._encoded_huge_string + # results = [] + self.banana.prepare() + # d.addCallback(results.append) + CHOMP = 4096 + for i in range(0, len(o), CHOMP): + self.banana.dataReceived(o[i:i+CHOMP]) + # print results + +import sys +from twisted.internet import reactor +from pyutil import benchutil +b = B() +for N in 10**3, 10**4, 10**5, 10**6, 10**7: + print "%8d" % N, + sys.stdout.flush() + benchutil.rep_bench(b.bench_huge_string_decode, N, b.setup_huge_string) diff -r 9adfe64ba138 foolscap/test/test_copyable.py --- a/foolscap/test/test_copyable.py Sun Mar 14 18:44:52 2010 -0700 +++ b/foolscap/test/test_copyable.py Thu Mar 25 02:53:09 2010 -0600 @@ -143,8 +143,8 @@ class Copyable(TargetMixin, unittest.Tes self.failUnlessEqual(f2.frames, []) self.failUnlessEqual(f2.tb, None) self.failUnlessEqual(f2.stack, []) - self.failUnless(f2.traceback.find("raise RuntimeError") != -1, - "no 'raise RuntimeError' in '%s'" % (f2.traceback,)) + self.failUnless(f2.traceback.find("RuntimeError") != -1, + "no 'RuntimeError' in '%s'" % (f2.traceback,)) def testFailure2(self): self.callingBroker.unsafeTracebacks = False