Lots of changes.

Renamed some things.
Added support for methods directly to bgenGenerator.py.
Completely reworked buffer, string and and structure types.

Tools/bgen/bgen/bgenGenerator.py

@@ -11,20 +11,28 @@
 INOUT = IN_OUT = "in-out"
 
 
-class Generator:
+class FunctionGenerator:
 
-	# XXX There are some funny things with the argument list.
+	def __init__(self, returntype, name, *argumentList):
-	# XXX It would be better to get rid of this and require
+		self.returntype = returntype
-	# XXX each argument to be a type object or a tuple of the form
+		self.name = name
-	# XXX (inoutmode, typeobject, argumentname)
+		self.argumentList = []
-	# XXX or possibly even a Variable() instance...
+		self.setreturnvar()
-
+		self.parseArgumentList(argumentList)
-	def __init__(self, *argumentList):
-		apply(self.parseArguments, argumentList)
 		self.prefix     = "XXX"    # Will be changed by setprefix() call
 		self.objecttype = "PyObject" # Type of _self argument to function
 		self.itselftype = None     # Type of _self->ob_itself, if defined
 
+	def setreturnvar(self):
+		if self.returntype:
+			self.rv = self.makereturnvar()
+			self.argumentList.append(self.rv)
+		else:
+			self.rv = None
+	
+	def makereturnvar(self):
+		return Variable(self.returntype, "_rv", OutMode)
+
 	def setprefix(self, prefix):
 		self.prefix = prefix
 
@@ -32,32 +40,12 @@ def setselftype(self, selftype, itselftype):
 		self.objecttype = selftype
 		self.itselftype = itselftype
 
-	def parseArguments(self, returnType, name, *argumentList):
-		if returnType:
-			self.rv = Variable(returnType, "_rv", OutMode)
-		else:
-			self.rv = None
-		self.name = name
-		self.argumentList = []
-		if self.rv:
-			self.argumentList.append(rv)
-		self.parseArgumentList(argumentList)
-
 	def parseArgumentList(self, argumentList):
-		from types import *
 		iarg = 0
-		for arg in argumentList:
+		for type, name, mode in argumentList:
-			# Arguments can either be:
-			# - a type
-			# - a tuple (type, [name, [mode]])
-			# - a variable
 			iarg = iarg + 1
-			if hasattr(arg, 'typeName'):
+			if name is None: name = "_arg%d" % iarg
-				arg = Variable(arg)
+			arg = Variable(type, name, mode)
-			elif type(arg) == TupleType:
-				arg = apply(Variable, arg)
-			if arg.name is None:
-				arg.name = "_arg%d" % iarg
 			self.argumentList.append(arg)
 
 	def reference(self, name = None):
@@ -67,6 +55,7 @@ def reference(self, name = None):
 		       name, self.prefix, self.name)
 
 	def generate(self):
+		print "-->", self.name
 		self.functionheader()
 		self.declarations()
 		self.getargs()
@@ -84,6 +73,7 @@ def functionheader(self):
 		Output("PyObject *_args;")
 		DedentLevel()
 		OutLbrace()
+		Output("PyObject *_res = NULL;")
 
 	def declarations(self):
 		for arg in self.argumentList:
@@ -98,7 +88,9 @@ def getargs(self):
 				continue
 			if arg.mode in (InMode, InOutMode):
 				fmt = fmt + arg.getargsFormat()
-				lst = lst + sep + arg.getargsArgs()
+				args = arg.getargsArgs()
+				if args:
+					lst = lst + sep + args
 		Output("if (!PyArg_ParseTuple(_args, \"%s\"%s))", fmt, lst)
 		IndentLevel()
 		Output("return NULL;")
@@ -111,7 +103,11 @@ def getargs(self):
 
 	def callit(self):
 		args = ""
-		sep = ",\n" + ' '*len("%s = %s(" % (self.rv.name, self.name))
+		if self.rv:
+			s = "%s = %s(" % (self.rv.name, self.name)
+		else:
+			s = "%s(" % self.name
+		sep = ",\n" + ' '*len(s)
 		for arg in self.argumentList:
 			if arg is self.rv:
 				continue
@@ -140,15 +136,34 @@ def returnvalue(self):
 				lst = lst + sep + arg.mkvalueArgs()
 		if fmt == "":
 			Output("Py_INCREF(Py_None);")
-			Output("return Py_None;");
+			Output("_res = Py_None;");
 		else:
-			Output("return Py_BuildValue(\"%s\"%s);", fmt, lst)
+			Output("_res = Py_BuildValue(\"%s\"%s);", fmt, lst)
+		tmp = self.argumentList[:]
+		tmp.reverse()
+		for arg in tmp:
+			if not arg: continue
+			if arg.flags == ErrorMode: continue
+			if arg.mode in (OutMode, InOutMode):
+				arg.mkvalueCleanup()
+		Output("return _res;")
 
 	def functiontrailer(self):
 		OutRbrace()
 
 
-class ManualGenerator(Generator):
+class MethodGenerator(FunctionGenerator):
+
+	def parseArgumentList(self, args):
+		a0, args = args[0], args[1:]
+		t0, n0, m0 = a0
+		if m0 != InMode:
+			raise ValueError, "method's 'self' must be 'InMode'"
+		self.itself = Variable(t0, "_self->ob_itself", SelfMode)
+		self.argumentList.append(self.itself)
+		FunctionGenerator.parseArgumentList(self, args)
+
+class ManualGenerator(FunctionGenerator):
 
 	def __init__(self, name, body):
 		self.name = name

Tools/bgen/bgen/bgenGeneratorGroup.py

@@ -13,7 +13,6 @@ def add(self, g):
 	def generate(self):
 		for g in self.generators:
 			g.generate()
-		Output()
 		Output("static PyMethodDef %s_methods[] = {", self.prefix)
 		IndentLevel()
 		for g in self.generators:
@@ -21,6 +20,7 @@ def generate(self):
 		Output("{NULL, NULL, 0}")
 		DedentLevel()
 		Output("};")
+		Output()
 
 
 def _test():

Tools/bgen/bgen/bgenObjectDefinition.py

@@ -57,18 +57,22 @@ def generate(self):
 		OutHeader2("End object type " + self.name)
 
 	def outputNew(self):
-		Output("static %s *%s_New(itself)", self.objecttype, self.prefix)
+		Output("static PyObject *%s_New(itself)", self.prefix)
 		IndentLevel()
 		Output("const %s %sitself;", self.itselftype, self.argref)
 		DedentLevel()
 		OutLbrace()
 		Output("%s *it;", self.objecttype)
+		self.outputCheckNewArg()
 		Output("it = PyObject_NEW(%s, &%s);", self.objecttype, self.typename)
 		Output("if (it == NULL) return NULL;")
 		Output("it->ob_itself = %sitself;", self.argref)
-		Output("return it;")
+		Output("return (PyObject *)it;")
 		OutRbrace()
 		Output()
+	
+	def outputCheckNewArg(self):
+		pass
 
 	def outputConvert(self):
 		Output("""\

Tools/bgen/bgen/bgenType.py

@@ -106,16 +106,53 @@ def mkvalueArgs(self, name):
 		"""
 		return name
 
+	def mkvalueCleanup(self, name):
+		"""Clean up if necessary after mkvalue().
+
+		This is normally empty; it may deallocate buffers etc.
+		"""
+		pass
+
+
+# Sometimes it's useful to define a type that's only usable as input or output parameter
+
+class InputOnlyMixIn:
+
+	"Mix-in class to boobytrap passOutput"
+
+	def passOutput(self):
+		raise RuntimeError, "this type can only be used for input parameters"
+
+class InputOnlyType(InputOnlyMixIn, Type):
+
+	"Same as Type, but only usable for input parameters -- passOutput is boobytrapped"
+
+class OutputOnlyMixIn:
+
+	"Mix-in class to boobytrap passInput"
+
+	def passInput(self):
+		raise RuntimeError, "this type can only be used for output parameters"
+
+class OutputOnlyType(OutputOnlyMixIn, Type):
+
+	"Same as Type, but only usable for output parameters -- passInput is boobytrapped"
+
 
 # A modest collection of standard C types.
 void = None
+char = Type("char", "c")
 short = Type("short", "h")
 int = Type("int", "i")
 long = Type("long", "l")
 float = Type("float", "f")
 double = Type("double", "d")
-stringptr = Type("char*", "s")
+
-char = Type("char", "c")
+
+# The most common use of character pointers is a null-terminated string.
+# For input, this is easy.  For output, and for other uses of char *,
+# see the various Buffer types below.
+stringptr = InputOnlyType("char*", "s")
 
 
 # Some Python related types.
@@ -124,20 +161,69 @@ def mkvalueArgs(self, name):
 # Etc.
 
 
-# Buffers are character arrays that may contain null bytes.
+class FakeType(InputOnlyType):
-# Their length is either Fixed or Sized (i.e. given by a separate argument).
 
-class SizedInputBuffer:
+	"""A type that is not represented in the Python version of the interface.
 
-	"Sized input buffer -- buffer size is an input parameter"
+	Instantiate with a value to pass in the call.
+	"""
 
-	def __init__(self, size = ''):
+	def __init__(self, substitute):
-		self.size = size
+		self.substitute = substitute
 
 	def declare(self, name):
-		Output("char *%s;", name)
+		pass
+
+	def getargsFormat(self):
+		return ""
+
+	def getargsArgs(self, name):
+		return None
+
+	def passInput(self, name):
+		return self.substitute
+
+
+class AbstractBufferType:
+	"""Buffers are character arrays that may contain null bytes.
+
+	There are a number of variants depending on:
+	- how the buffer is allocated (for output buffers), and
+	- whether and how the size is passed into and/or out of the called function.
+
+	The AbstractbufferType only serves as a placeholder for this doc string.
+	"""
+
+	def declare(self, name):
+		self.declareBuffer(name)
+		self.declareSize(name)
+
+
+class FixedBufferType(AbstractBufferType):
+
+	"""Fixed size buffer -- passed without size information.
+
+	Instantiate with the size as parameter.
+	THIS IS STILL AN ABSTRACT BASE CLASS -- DO NOT INSTANTIATE.
+	"""
+
+	def __init__(self, size):
+		self.size = str(size)
+
+	def declareSize(self, name):
 		Output("int %s__len__;", name)
 
+
+class FixedInputBufferType(InputOnlyMixIn, FixedBufferType):
+
+	"""Fixed size input buffer -- passed without size information.
+
+	Instantiate with the size as parameter.
+	"""
+
+	def declareBuffer(self, name):
+		Output("char *%s;", name)
+
 	def getargsFormat(self):
 		return "s#"
 
@@ -145,126 +231,221 @@ def getargsArgs(self, name):
 		return "&%s, &%s__len__" % (name, name)
 
 	def getargsCheck(self, name):
-		pass
+		Output("if (%s__len__ != %s)", name, self.size)
-
-	def passInput(self, name):
-		return "%s, %s__len__" % (name, name)
-
-
-class FixedInputBuffer(SizedInputBuffer):
-
-	"Fixed input buffer -- buffer size is a constant"
-
-	def getargsCheck(self, name):
-		Output("if (%s__len__ != %s)", name, str(self.size))
 		OutLbrace()
-		Output('PyErr_SetString(PyExc_TypeError, "bad string length");')
+		Output('PyErr_SetString(PyExc_TypeError, "buffer length should be %s");',
-		Output('return NULL;')
+		       self.size)
+		Output('return NULL;') # XXX should do a goto
 		OutRbrace()
 
 	def passInput(self, name):
 		return name
 
 
-class RecordBuffer(FixedInputBuffer):
+class FixedOutputBufferType(OutputOnlyMixIn, FixedBufferType):
 
-	"Like fixed input buffer -- but declared as a record type pointer"
+	"""Fixed size output buffer -- passed without size information.
 
-	def __init__(self, type):
+	Instantiate with the size as parameter.
-		self.type = type
+	"""
-		self.size = "sizeof(%s)" % type
 
-	def declare(self, name):
+	def declareBuffer(self, name):
-		Output("%s *%s;", self.type, name)
+		Output("char %s[%s];", name, self.size)
-		Output("int %s__len__;", name)
-	
-
-class SizedOutputBuffer:
-
-	"Sized output buffer -- buffer size is an input-output parameter"
-
-	def __init__(self, maxsize):
-		self.maxsize = maxsize
-
-	def declare(self, name):
-		Output("char %s[%s];", name, str(self.maxsize))
-		Output("int %s__len__ = %s;", name, str(self.maxsize))
 
 	def passOutput(self, name):
-		return "%s, &%s__len__" % (name, name)
+		return name
-
-	def errorCheck(self, name):
-		pass
 
 	def mkvalueFormat(self):
 		return "s#"
 
+	def mkvalueArgs(self):
+		return "%s, %s" % (name, self.size)
+
+
+class StructBufferType(FixedBufferType):
+
+	"""Structure buffer -- passed as a structure pointer.
+
+	Instantiate with the struct type as parameter.
+	"""
+
+	def __init__(self, type):
+		FixedBufferType.__init__(self, "sizeof(%s)" % type)
+		self.type = type
+
+
+class StructInputBufferType(StructBufferType, FixedInputBufferType):
+
+	"""Fixed size input buffer -- passed as a pointer to a structure.
+
+	Instantiate with the struct type as parameter.
+	"""
+
+	def declareBuffer(self, name):
+		Output("%s *%s;", self.type, name)
+
+
+class StructByValueBufferType(StructInputBufferType):
+
+	"""Fixed size input buffer -- passed as a structure BY VALUE.
+
+	Instantiate with the struct type as parameter.
+	"""
+
+	def passInput(self, name):
+		return "*%s" % name
+
+
+class StructOutputBufferType(StructBufferType, FixedOutputBufferType):
+
+	"""Fixed size output buffer -- passed as a pointer to a structure.
+
+	Instantiate with the struct type as parameter.
+	"""
+
+	def declareBuffer(self, name):
+		Output("%s %s;", self.type, name)
+
+	def passOutput(self, name):
+		return "&%s" % name
+
+	def mkvalueArgs(self, name):
+		return "(char *)&%s" % name
+
+
+class VarInputBufferType(InputOnlyMixIn, FixedInputBufferType):
+
+	"""Input buffer -- passed as (buffer, size).
+
+	Instantiate without parameters.
+	"""
+
+	def __init__(self):
+		pass
+
+	def getargsCheck(self, name):
+		pass
+
+	def passInput(self, name):
+		return "%s, %s__len__" % (name, name)
+
+
+class StackOutputBufferType(OutputOnlyMixIn, FixedOutputBufferType):
+
+	"""Output buffer -- passed as (buffer, size).
+
+	Instantiate with the buffer size as parameter.
+	"""
+
+	def passOutput(self, name):
+		return "%s, %s" % (name, self.size)
+
+
+class VarStackOutputBufferType(StackOutputBufferType):
+
+	"""Output buffer allocated on the stack -- passed as (buffer, &size).
+
+	Instantiate with the buffer size as parameter.
+	"""
+
+	def declareSize(self, name):
+		Output("int %s__len__ = %s;", name, self.size)
+
+	def passOutput(self, name):
+		return "%s, &%s__len__" % (name, name)
+
 	def mkvalueArgs(self, name):
 		return "%s, %s__len__" % (name, name)
 
 
-class VarSizedOutputBuffer(SizedOutputBuffer):
+class VarVarStackOutputBufferType(VarStackOutputBufferType):
 
-	"Variable Sized output buffer -- buffer size is an input and an output parameter"
+	"""Output buffer allocated on the stack -- passed as (buffer, size, &size).
 
-	def getargsFormat(self):
+	Instantiate with the buffer size as parameter.
-		return "i"
+	"""
-		
-	def getargsArgs(self, name):
-		return "&%s__len__" % name
-		
-	def getargsCheck(self, name):
-		pass
 
 	def passOutput(self, name):
 		return "%s, %s__len__, &%s__len__" % (name, name, name)
 
 
-class FixedOutputBuffer:
+class HeapOutputBufferType(FixedOutputBufferType):
 
-	"Fixed output buffer -- buffer size is a constant"
+	"""Output buffer allocated on the heap -- passed as (buffer, size).
 
-	def __init__(self, size):
+	Instantiate without parameters.
-		self.size = size
+	Call from Python with buffer size.
+	"""
 
-	def declare(self, name):
+	def __init__(self):
-		Output("char %s[%s];", name, str(self.size))
-
-	def passOutput(self, name):
-		return name
-
-	def errorCheck(self, name):
 		pass
 
-	def mkvalueFormat(self):
+	def declareBuffer(self, name):
-		return "s#"
+		Output("char *%s;", name)
+
+	def getargsFormat(self):
+		return "i"
+
+	def getargsArgs(self, name):
+		return "&%s__len__" % name
+
+	def getargsCheck(self, name):
+		Output("if ((%s = malloc(%s__len__)) == NULL) goto %s__error__;",
+		             name,       name,                     name)
+
+	def passOutput(self, name):
+		return "%s, %s__len__" % (name, name)
 
 	def mkvalueArgs(self, name):
-		return "%s, %s" % (name, str(self.size))
+		return "%s, %s__len__" % (name, name)
+
+	def mkvalueCleanup(self, name):
+		Output("free(%s);", name)
+		DedentLevel()
+		Output(" %s__error__: ;", name);
+		IndentLevel()
 
 
-# Strings are character arrays terminated by a null byte.
+class VarHeapOutputBufferType(HeapOutputBufferType):
-# For input, this is covered by stringptr.
-# For output, there are again two variants: Fixed (size is a constant
-# given in the documentation) or Sized (size is given by a variable).
-# (Note that this doesn't cover output parameters in which a string
-# pointer is returned.)
 
-class SizedOutputString:
+	"""Output buffer allocated on the heap -- passed as (buffer, &size).
 
-	"Null-terminated output string -- buffer size is an input parameter"
+	Instantiate without parameters.
-
+	Call from Python with buffer size.
-	def __init__(self, bufsize):
+	"""
-		self.bufsize = bufsize
-
-	def declare(self, name):
-		Output("char %s[%s];", name, str(self.bufsize))
 
 	def passOutput(self, name):
-		return "%s, %s" % (name, str(self.bufsize))
+		return "%s, &%s__len__" % (name, name)
 
-	def errorCheck(self, name):
+
-		pass
+class VarVarHeapOutputBufferType(VarHeapOutputBufferType):
+
+	"""Output buffer allocated on the heap -- passed as (buffer, size, &size).
+
+	Instantiate without parameters.
+	Call from Python with buffer size.
+	"""
+
+	def passOutput(self, name):
+		return "%s, %s__len__, &%s__len__" % (name, name, name)
+
+
+class StringBufferType:
+
+	"""Mix-in class to create various string buffer types.
+
+	Strings are character arrays terminated by a null byte.
+	For input, this is already covered by stringptr.
+	For output, there are again three variants:
+	- Fixed (size is a constant given in the documentation),
+	- Stack (size is passed to the C function but we decide on a size at
+	  code generation time so we can still allocate on the heap), or
+	- Heap (size is passed to the C function and we let the Python caller
+	  pass a size.
+	(Note that this doesn't cover output parameters in which a string
+	pointer is returned.  These are actually easier (no allocation) but far
+	less common.  I'll write the classes when there is demand.)
+	"""
 
 	def mkvalueFormat(self):
 		return "s"
@@ -273,63 +454,88 @@ def mkvalueArgs(self, name):
 		return name
 
 
-class FixedOutputString(SizedOutputString):
+class FixedOutputStringType(StringBufferType, FixedOutputBufferType):
 
-	"Null-terminated output string -- buffer size is a constant"
+	"""Null-terminated output string -- passed without size.
+
+	Instantiate with buffer size as parameter.
+	"""
+
+
+class StackOutputStringType(StringBufferType, StackOutputBufferType):
+
+	"""Null-terminated output string -- passed as (buffer, size).
+
+	Instantiate with buffer size as parameter.
+	"""
+
+class HeapOutputStringType(StringBufferType, HeapOutputBufferType):
+
+	"""Null-terminated output string -- passed as (buffer, size).
+
+	Instantiate without parameters.
+	Call from Python with buffer size.
+	"""
+
+
+class OpaqueType(Type):
+
+	"""A type represented by an opaque object type, always passed by address.
+
+	Instantiate with the type name, and optional an object type name whose
+	New/Convert functions will be used.
+	"""
+
+	def __init__(self, name, sameAs = None):
+		self.typeName = name
+		self.sameAs = sameAs or name
+
+	def getargsFormat(self):
+		return 'O&'
+
+	def getargsArgs(self, name):
+		return "%s_Convert, &%s" % (self.sameAs, name)
+
+	def passInput(self, name):
+		return "&%s" % name
+
+	def mkvalueFormat(self):
+		return 'O&'
+
+	def mkvalueArgs(self, name):
+		return "%s_New, &%s" % (self.sameAs, name)
+
+
+class OpaqueByValueType(OpaqueType):
+
+	"""A type represented by an opaque object type, on input passed BY VALUE.
+
+	Instantiate with the type name, and optional an object type name whose
+	New/Convert functions will be used.
+	"""
+
+	def passInput(self, name):
+		return name
+
+	def mkvalueArgs(self, name):
+		return "%s_New, %s" % (self.sameAs, name)
+
+
+class OpaqueArrayType(OpaqueByValueType):
+
+	"""A type represented by an opaque object type, with ARRAY passing semantics.
+
+	Instantiate with the type name, and optional an object type name whose
+	New/Convert functions will be used.
+	"""
+
+	def getargsArgs(self, name):
+		return "%s_Convert, &%s" % (self.sameAs, name)
 
 	def passOutput(self, name):
 		return name
 
 
-class StructureByValue:
-
-	"Structure passed by value -- mapped to a Python string (for now)"
-
-	def __init__(self, typeName):
-		self.typeName = typeName
-
-	def declare(self, name):
-		n = len(self.typeName)
-		Output("%-*s %s;",        n, self.typeName, name)
-		Output("%-*s*%s__str__;", n, "char",        name)
-		Output("%-*s %s__len__;", n, "int",         name)
-
-	def getargsFormat(self):
-		return "s#"
-
-	def getargsArgs(self, name):
-		return "&%s__str__, &%s__len__" % (name, name)
-
-	def getargsCheck(self, name):
-		Output("if (%s__len__ != sizeof %s)", name, name)
-		IndentLevel()
-		Output('PyErr_SetString(PyExc_TypeError, "bad structure length");')
-		DedentLevel()
-		Output("memcpy(&%s, %s__str__, %s__len__);",
-		       name, name, name)
-
-	def passInput(self, name):
-		return "%s" % name
-
-	def passOutput(self, name):
-		return "&%s" % name
-
-	def errorCheck(self, name):
-		pass
-
-	def mkvalueFormat(self):
-		return "s#"
-
-	def mkvalueArgs(self, name):
-		return "(char *)&%s, (int)sizeof %s" % (name, name)
-
-
-class StructureByAddress(StructureByValue):
-
-	def passInput(self, name):
-		return "&%s" % name
-
-
 class Variable:
 
 	"""A Variable holds a type, a name, a transfer mode and flags.
@@ -397,3 +603,7 @@ def mkvalueFormat (self):
 	def mkvalueArgs(self):
 		"""Call the type's mkvalueArgs method."""
 		return self.type.mkvalueArgs(self.name)
+
+	def mkvalueCleanup(self):
+		"""Call the type's mkvalueCleanup method."""
+		return self.type.mkvalueCleanup(self.name)