RMUL2025/lib/cmsis_5/CMSIS/DSP/Testing/TestScripts/CodeGen.py

896 lines
31 KiB
Python

import TestScripts.Parser
import sys
import os.path
import math
groupCode="""class %s : public Client::Group
{
public:
%s(Testing::testID_t id):Client::Group(id)
%s
{
%s
}
private:
%s
};
"""
suiteCode="""
#include \"%s.h\"
%s::%s(Testing::testID_t id):Client::Suite(id)
{
%s
}
"""
def decodeHex(hexstr,bits,mask):
value = int(hexstr,16) & mask
if value & (1 << (bits-1)):
value -= 1 << bits
return value
def createMissingDir(destPath):
theDir=os.path.normpath(os.path.dirname(destPath))
if not os.path.exists(theDir):
os.makedirs(theDir)
class CodeGen:
""" Code generation from the parsed description file
Generate include files, cpp files, and .txt files for test control
Generation depends on the mode (fpga or semihosting)
"""
def __init__(self,patternDir,paramDir,fpga):
""" Create a CodeGen object
Args:
patternDir (str) : where pattern must be read
Used to generate include file in fpag mode
fpga (bool) : false in semihosting mode
Raises:
Nothing
Returns:
CodeGen : CodeGen object
"""
self._fpga = fpga
self._patternDir = patternDir
self._paramDir = paramDir
self._currentPaths = [self._patternDir]
self._currentParamPaths = [self._paramDir]
self._alignment=8
def _genGroup(self,root,fi):
""" Generate header definition for a group of tests
Args:
root (TreeElem) : group object to use
fi (file) : Header file (TestDesc.h)
Where to write include definitions for the classes
Raises:
Nothing
Returns:
Nothing
"""
# Get the CPP class name
theClass = root.data["class"]
# Get the group ID
theId = root.id
varInit = ""
testInit = ""
testDecl = ""
# Generate initializer for the constructors
# All the member variables corresponding to the
# suites and groups contained in this group.
i = 1
for c in root:
theData = c.data
theMemberId = c.id
if not theData["deprecated"]:
theMemberClass = theData["class"]
theMemberVar = theMemberClass + "Var"
varInit += (",%s(%d)\n" % (theMemberVar,theMemberId))
i = i + 1
# Generate initialization code for the constructor
i = 1
for c in root:
theData = c.data
if theData["deprecated"]:
testInit += "this->addContainer(NULL);"
else:
theMemberClass = theData["class"]
theMemberVar = theMemberClass+"Var"
testInit += "this->addContainer(&%s);\n" % (theMemberVar)
i = i + 1
# Generate member variables for the groups and the suites
# contained in this group.
i = 1
for c in root:
theData = c.data
if not theData["deprecated"]:
theMemberClass = theData["class"]
theMemberVar = theMemberClass+"Var"
theMemberId = c.id
testDecl += "%s %s;\n" % (theMemberClass,theMemberVar)
i = i + 1
fi.write(groupCode % (theClass,theClass,varInit,testInit,testDecl) )
def _genSuite(self,root,thedir,sourceFile):
""" Generate source definition for suite
Args:
root (TreeElem) : suite object to use
fi (file) : Source file (TestDesc.cpp)
Where to write source definitions for the classes
Raises:
Nothing
Returns:
Nothing
"""
# Get the CPP class
theClass = root.data["class"]
testInit = ""
testDecl=""
declared={}
# Generate constructor for the class
# Test functions are added in the constructor.
# Keep track of the list of test functions
# (since a test function can be reused by several tests)
for c in root:
theData = c.data
theId = c.id
theTestName = theData["class"]
if not theData["deprecated"]:
testInit += "this->addTest(%d,(Client::test)&%s::%s);\n" % (theId,theClass,theTestName)
# To be able to deprecated tests without having to change the code
# we dump const declaration even for deprecated functions
if theTestName not in declared:
testDecl += "void %s();\n" % (theTestName)
declared[theTestName] = 1
# Generate the suite constructor.
sourceFile.write(suiteCode % (theClass,theClass,theClass,testInit) )
# Generate specific header file for this suite
with open(os.path.join(thedir,"%s_decl.h" % theClass),"w") as decl:
# Use list of test functions to
# declare them in the header.
# Since we don't want to declare the functino several times,
# that's why we needed to keep track of functions in the code above:
# to remove redundancies since the function
# can appear several time in the root object.
decl.write(testDecl)
# Generate the pattern ID for patterns
# defined in this suite
decl.write("\n// Pattern IDs\n")
i = 0
for p in root.patterns:
newId = "static const int %s=%d;\n" % (p[0],i)
decl.write(newId)
i = i + 1
# Generate output ID for the output categories
# defined in this suite
decl.write("\n// Output IDs\n")
i = 0
for p in root.outputs:
newId = "static const int %s=%d;\n" % (p[0],i)
decl.write(newId)
i = i + 1
# Generate test ID for the test define din this suite
i = 0
decl.write("\n// Test IDs\n")
for c in root:
theData = c.data
theId = c.id
# To be able to deprecated tests without having to change the code
# we dump const declaration even for deprecated functions
#if not theData["deprecated"]:
theTestName = theData["class"]
defTestID = "static const int %s_%d=%d;\n" % (theTestName.upper(),theId,theId)
decl.write(defTestID)
i = i + 1
def _genCode(self,root,dir,sourceFile,headerFile):
""" Generate code for the tree of tests
Args:
root (TreeElem) : root object containing the tree
sourceFile (file) : Source file (TestDesc.cpp)
Where to write source definitions for the classes
headerFile (file) : Heade5 file (TestDesc.h)
Where to write header definitions for the classes
Raises:
Nothing
Returns:
Nothing
"""
deprecated = root.data["deprecated"]
if not deprecated:
if root.kind == TestScripts.Parser.TreeElem.GROUP:
# For a group, first iterate on the children then on the
# parent because the parent definitino in the header is
# dependent on the children definition (member varaiables)
# So children must gbe defined before.
for c in root:
self._genCode(c,dir,sourceFile,headerFile)
self._genGroup(root,headerFile)
# For a suite, we do not need to recurse since it is a leaf
# of the tree of tests and is containing a list of tests to
# generate
if root.kind == TestScripts.Parser.TreeElem.SUITE:
self._genSuite(root,dir,sourceFile)
def getSuites(self,root,listOfSuites):
""" Extract the list of all suites defined in the tree
Args:
root (TreeElem) : root object containing the tree
listOfSuites (list) : list of suites
Raises:
Nothing
Returns:
list : list of suites
"""
deprecated = root.data["deprecated"]
if not deprecated:
theClass = root.data["class"]
if root.kind == TestScripts.Parser.TreeElem.SUITE:
# Append current suite to the list and return the result
listOfSuites.append(theClass)
return(listOfSuites)
elif root.kind == TestScripts.Parser.TreeElem.GROUP:
# Recurse on the children to get the listOfSuite.
# getSuites is appending to the list argument.
# So the line listOfSuites = self.getSuites(c,listOfSuites)
# is a fold.
for c in root:
listOfSuites = self.getSuites(c,listOfSuites)
return(listOfSuites)
else:
return(listOfSuites)
else:
return(listOfSuites)
def _genText(self,root,textFile):
""" Generate the text file which is driving the tests in semihosting
Format of file is:
node kind (group, suite or test)
node id (group id, suite id, test id)
y or n to indicate if this node has some parameters
if y there is the parameter ID
y or n to indicate if entering this node is adding a new
folder to the path used to get input files
if y, the path for the node is added.
Then, if node is a suite, description of the suite is generated.
Number of parameters or 0
It is the number of arguments used by function setUp (arity).
It is not the number of samples in a paramater file
The number of samples is always a multiple
For instance if width and height are the parameters then number
of parameters (arity) is 2.
But each parameter file may contain several combination of (width,height)
So the lenght of those files will be a multiple of 2.
The number of patterns path is generated
The patterns names are generated
The number of output names is generated
The output names are generated
The number of parameter path is generated
The parameter description are generated
p
path if a path
g
gen data if a generator
If a generator:
Generator kind (only 1 = cartesian product generator)
Number of input samples (number of values to encode all parameters)
Number of output samples (number of generated values)
Number of dimensions
For each dimension
Length
Samples
For instance, with A=[1,2] and B=[1,2,3].
Input dimension is 1+2 + 1 + 3 = 7 (number of values needed to encode the two lists
with their length).
Number of output dimensions is 2 * 3 = 6
So, for a test (which is also a TreeElement in the structure),
only node kind and node id are generated followed by
param description and n for folder (since ther is never any folder)
In the test description file, there is never any way to change the pattern
folder for a test.
Args:
root (TreeElem) : root object containing the tree
textFile (file) : where to write the driving description
Raises:
Nothing
Returns:
Nothing
"""
deprecated = root.data["deprecated"]
if not deprecated:
textFile.write(str(root.kind))
textFile.write(" ")
textFile.write(str(root.id))
textFile.write("\n")
if "PARAMID" in root.data:
if root.parameterToID:
textFile.write("y\n")
paramId = root.parameterToID[root.data["PARAMID"]]
textFile.write(str(paramId))
textFile.write("\n")
elif root.parent.parameterToID:
textFile.write("y\n")
paramId = root.parent.parameterToID[root.data["PARAMID"]]
textFile.write(str(paramId))
textFile.write("\n")
else:
textFile.write("n\n")
else:
textFile.write("n\n")
# Always dump even if there is no path for a test
# so for a test it will always be n
# It is done to simplify parsing on C side
if root.path:
textFile.write("y\n")
textFile.write(root.path)
textFile.write('\n')
else:
textFile.write("n\n")
if root.kind == TestScripts.Parser.TreeElem.SUITE:
# Here we dump the number of parameters used
# for the tests / benchmarks
if root.params:
textFile.write("%d\n" % len(root.params.full))
else:
textFile.write("0\n")
# Generate patterns path
textFile.write("%d\n" % len(root.patterns))
for (patid,patpath) in root.patterns:
#textFile.write(patid)
#textFile.write("\n")
textFile.write(patpath.strip())
textFile.write("\n")
# Generate output names
textFile.write("%d\n" % len(root.outputs))
for (outid,outname) in root.outputs:
#textFile.write(patid)
#textFile.write("\n")
textFile.write(outname.strip())
textFile.write("\n")
# Generate parameters path or generator
textFile.write("%d\n" % len(root.parameters))
for (paramKind,parid,parpath) in root.parameters:
if paramKind == TestScripts.Parser.TreeElem.PARAMFILE:
textFile.write("p\n")
textFile.write(parpath.strip())
textFile.write("\n")
# Generator kind (only 1 = cartesian product generator)
# Number of input samples (dimensions + vectors)
# Number of samples generated when run
# Number of dimensions
# For each dimension
# Length
# Samples
if paramKind == TestScripts.Parser.TreeElem.PARAMGEN:
textFile.write("g\n")
dimensions = len(parpath)
nbOutputSamples = 1
nbInputSamples = 0
for c in parpath:
nbOutputSamples = nbOutputSamples * len(c["INTS"])
nbInputSamples = nbInputSamples + len(c["INTS"]) + 1
textFile.write("1")
textFile.write("\n")
textFile.write(str(nbInputSamples))
textFile.write("\n")
textFile.write(str(nbOutputSamples))
textFile.write("\n")
textFile.write(str(dimensions))
textFile.write("\n")
for c in parpath:
textFile.write(str(len(c["INTS"])))
textFile.write("\n")
for d in c["INTS"]:
textFile.write(str(d))
textFile.write("\n")
# Iterate on the children
for c in root:
self._genText(c,textFile)
def _write64(self,v,f):
""" Write four integers into a C char array to represent word32
It is used to dump input patterns in include files
or test drive in include file
Args:
v (int) : The int64 to write
f (file) : the opended file
Raises:
Nothing
Returns:
Nothing
"""
a=[0,0,0,0,0,0,0,0]
a[0]= v & 0x0FF
v = v >> 8
a[1]= v & 0x0FF
v = v >> 8
a[2]= v & 0x0FF
v = v >> 8
a[3]= v & 0x0FF
v = v >> 8
a[4]= v & 0x0FF
v = v >> 8
a[5]= v & 0x0FF
v = v >> 8
a[6]= v & 0x0FF
v = v >> 8
a[7]= v & 0x0FF
v = v >> 8
f.write("%d,%d,%d,%d,%d,%d,%d,%d,\n" % (a[0],a[1],a[2],a[3],a[4],a[5],a[6],a[7]))
def _write32(self,v,f):
""" Write four integers into a C char array to represent word32
It is used to dump input patterns in include files
or test drive in include file
Args:
v (int) : The int32 to write
f (file) : the opended file
Raises:
Nothing
Returns:
Nothing
"""
a=[0,0,0,0]
a[0]= v & 0x0FF
v = v >> 8
a[1]= v & 0x0FF
v = v >> 8
a[2]= v & 0x0FF
v = v >> 8
a[3]= v & 0x0FF
v = v >> 8
f.write("%d,%d,%d,%d,\n" % (a[0],a[1],a[2],a[3]))
def _write16(self,v,f):
""" Write 2 integers into a C char array to represent word32
It is used to dump input patterns in include files
or test drive in include file
Args:
v (int) : The int3 to write
f (file) : the opended file
Raises:
Nothing
Returns:
Nothing
"""
a=[0,0]
a[0]= v & 0x0FF
v = v >> 8
a[1]= v & 0x0FF
f.write("%d,%d,\n" % (a[0],a[1]))
def _write8(self,v,f):
""" Write 4 integers into a C char array to represent word8
It is used to dump input patterns in include files
or test drive in include file
Args:
v (int) : The int to write
f (file) : the opended file
Raises:
Nothing
Returns:
Nothing
"""
a=[0]
a[0]= v & 0x0FF
f.write("%d,\n" % (a[0]))
def _writeString(self,v,f):
""" Write a C string into a C char array to represent as a list of int
It is used to dump input patterns in include files
or test drive in include file
Args:
v (str) : The string to write
f (file) : the opended file
Raises:
Nothing
Returns:
Nothing
"""
for c in v:
f.write("'%c'," % c)
f.write("'\\0',\n")
def convertToInt(self,k,s):
v = 0
if k == "D":
v = decodeHex(s,64,0x0FFFFFFFFFFFFFFFF)
if k == "W":
v = decodeHex(s,32,0x0FFFFFFFF)
if k == "H":
v = decodeHex(s,16,0x0FFFF)
if k == "B":
v = decodeHex(s,8,0x0FF)
return(v)
def addPattern(self,includeFile,path):
""" Add a pattern to the include file
It is writing sequence of int into a C char array
to represent the pattern.
Assuming the C chr array is aligned, pattern offset are
aligned too.
So, some padding with 0 may also be generated after the patterm
Args:
includeFile (file) : Opened include file
path (str) : Path to file containing the data
Raises:
Nothing
Returns:
(int,int) : The pattern offset in the array and the number of samples
"""
# Current offset in the array which is the offset for the
# pattern being added to this array
returnOffset = self._offset
# Read the pattern for the pattern file
with open(path,"r") as pat:
# Read pattern word size (B,H or W for 8, 16 or 32 bits)
# Patterns are containing data in hex format (big endian for ARM)
# So there is no conversion to do.
# Hex data is read and copied as it is in the C array
k =pat.readline().strip()
sampleSize=1
if k == 'D':
sampleSize = 8
if k == 'W':
sampleSize = 4
if k == 'H':
sampleSize = 2
# Read number of samples
nbSamples = int(pat.readline().strip())
# Compute new offset based on pattern length only
newOffset = self._offset + sampleSize * nbSamples
# Compute padding due to alignment constraint
pad = self._alignment*math.ceil(newOffset / self._alignment) - newOffset
# New offset in the array
self._offset=newOffset + pad
# Generate all samples into the C array
for i in range(nbSamples):
# In pattern file we have a comment giving the
# true value (for instance float)
# and then a line giving the hex data
# We Ignore comment
pat.readline()
# Then we read the Value
v = self.convertToInt(k,pat.readline())
# Depending on the word size, this hex must be writen to
# the C array as 4,2 or 1 number.
if k == 'D':
self._write64(v,includeFile)
if k == 'W':
self._write32(v,includeFile)
if k == 'H':
self._write16(v,includeFile)
if k == 'B':
self._write8(v,includeFile)
#includeFile.write("%d,\n" % v)
# Add the padding to the pattern
for i in range(pad):
includeFile.write("0,\n")
return(returnOffset,nbSamples)
def addParameter(self,includeFile,path):
""" Add a parameter array to the include file
It is writing sequence of int into a C char array
to represent the pattern.
Assuming the C chr array is aligned, pattern offset are
aligned too.
So, some padding with 0 may also be generated after the patterm
Args:
includeFile (file) : Opened include file
path (str) : Path to file containing the data
Raises:
Nothing
Returns:
(int,int) : The pattern offset in the array and the number of samples
"""
# Current offset in the array which is the offset for the
# pattern being added to this array
returnOffset = self._offset
# Read the pattern for the pattern file
with open(path,"r") as pat:
# Read pattern word size (B,H or W for 8, 16 or 32 bits)
# Patterns are containing data in hex format (big endian for ARM)
# So there is no conversion to do.
# Hex data is read and copied as it is in the C array
sampleSize = 4
# Read number of samples
nbSamples = int(pat.readline().strip())
# Compute new offset based on pattern length only
newOffset = self._offset + sampleSize * nbSamples
# Compute padding due to alignment constraint
pad = self._alignment*math.ceil(newOffset / self._alignment) - newOffset
# New offset in the array
self._offset=newOffset + pad
# Generate all samples into the C array
for i in range(nbSamples):
# In pattern file we have a comment giving the
# true value (for instance float)
# and then a line giving the hex data
# Then we read the Value
v = int(pat.readline().strip(),0)
# Depending on the word size, this hex must be writen to
# the C array as 4,2 or 1 number.
self._write32(v,includeFile)
# Add the padding to the pattern
for i in range(pad):
includeFile.write("0,\n")
return(returnOffset,nbSamples)
def _genDriver(self,root,driverFile,includeFile):
""" Generate the driver file and the pattern file
Args:
root (TreeElement) : Tree of test descriptions
driverFile (file) : where to generate C array for test descriptions
includeFile (file) : where to generate C array for patterns
Raises:
Nothing
Returns:
Nothing
"""
#if root.parent:
# print(root.parent.data["message"])
#print(" ",root.data["message"])
#print(self._currentPaths)
deprecated = root.data["deprecated"]
# We follow a format quite similar to what is generated in _genText
# for the text description
# But here we are using an offset into the pattern array
# rather than a path to a pattern file.
# It is the only difference
# But for outputs we still need the path so the logic is the same
# Path for output is required to be able to extract data from the stdout file
# and know where the data must be written to.
# Please refer to comments of _genText for description of the format
oldPath = self._currentPaths.copy()
oldParamPath = self._currentParamPaths.copy()
if not deprecated:
# We write node kind and node id
self._write32(root.kind,driverFile)
self._write32(root.id,driverFile)
if "PARAMID" in root.data:
if root.parameterToID:
driverFile.write("'y',")
paramId = root.parameterToID[root.data["PARAMID"]]
self._write32(int(paramId),driverFile)
elif root.parent.parameterToID:
driverFile.write("'y',")
paramId = root.parent.parameterToID[root.data["PARAMID"]]
self._write32(int(paramId),driverFile)
else:
driverFile.write("'n',")
else:
driverFile.write("'n',")
# We write a folder path
# if folder changed in test description file
# Always dumped for a test even if no path for
# a test. So a test will always have n
# It is done to simplify parsing on C side
if root.path:
self._currentPaths.append(root.path)
self._currentParamPaths.append(root.path)
driverFile.write("'y',")
self._writeString(root.path,driverFile)
else:
driverFile.write("'n',\n")
if root.kind == TestScripts.Parser.TreeElem.SUITE:
# Number of parameters
if root.params:
self._write32(len(root.params.full),driverFile)
else:
self._write32(0,driverFile)
# Patterns offsets are written
# and pattern length since the length is not available in a file
# like for semihosting version
self._write32(len(root.patterns),driverFile)
for (patid,patpath) in root.patterns:
temp = self._currentPaths.copy()
temp.append(patpath)
includeFile.write("// " + os.path.join(*temp) + "\n")
offset,nbSamples=self.addPattern(includeFile,os.path.join(*temp))
#driverFile.write(patpath)
#driverFile.write("\n")
self._write32(offset,driverFile)
self._write32(nbSamples,driverFile)
# Generate output names
self._write32(len(root.outputs),driverFile)
for (outid,outname) in root.outputs:
#textFile.write(patid)
#textFile.write("\n")
self._writeString(outname.strip(),driverFile)
# Parameters offsets are written
# and parameter length since the length is not available in a file
# like for semihosting version
self._write32(len(root.parameters),driverFile)
for (paramKind,parid,parpath) in root.parameters:
if paramKind == TestScripts.Parser.TreeElem.PARAMFILE:
temp = self._currentParamPaths.copy()
temp.append(parpath)
includeFile.write("// " + os.path.join(*temp) + "\n")
offset,nbSamples=self.addParameter(includeFile,os.path.join(*temp))
#driverFile.write(patpath)
#driverFile.write("\n")
driverFile.write("'p',")
self._write32(offset,driverFile)
self._write32(nbSamples,driverFile)
# TO DO
if paramKind == TestScripts.Parser.TreeElem.PARAMGEN:
temp = self._currentParamPaths.copy()
includeFile.write("// " + os.path.join(*temp) + "\n")
driverFile.write("'g',")
dimensions = len(parpath)
nbOutputSamples = 1
nbInputSamples = 0
for c in parpath:
nbOutputSamples = nbOutputSamples * len(c["INTS"])
nbInputSamples = nbInputSamples + len(c["INTS"]) + 1
self._write32(1,driverFile)
self._write32(nbInputSamples,driverFile)
self._write32(nbOutputSamples,driverFile)
self._write32(dimensions,driverFile)
for c in parpath:
self._write32(len(c["INTS"]),driverFile)
for d in c["INTS"]:
self._write32(d,driverFile)
# Recurse on the children
for c in root:
self._genDriver(c,driverFile,includeFile)
self._currentPaths = oldPath.copy()
self._currentParamPaths = oldParamPath.copy()
def genCodeForTree(self,root,benchMode):
""" Generate all files from the trees of tests
Args:
root (TreeElement) : Tree of test descriptions
Raises:
Nothing
Returns:
Nothing
"""
# Get a list of all suites contained in the tree
suites = self.getSuites(root,[])
src = "GeneratedSource"
header = "GeneratedInclude"
if benchMode:
src += "Bench"
header += "Bench"
# Generate .cpp and .h files neded to run the tests
with open("%s/TestDesc.cpp" % src,"w") as sourceFile:
with open("%s/TestDesc.h" % header,"w") as headerFile:
headerFile.write("#include \"Test.h\"\n")
headerFile.write("#include \"Pattern.h\"\n")
sourceFile.write("#include \"Test.h\"\n")
for s in suites:
headerFile.write("#include \"%s.h\"\n" % s)
self._genCode(root,"%s" % header,sourceFile,headerFile)
# Generate a driver file for semihosting
# (always generated for debug purpose since it is the reference format)
with open("TestDesc.txt","w") as textFile:
self._genText(root,textFile)
# If fpga mode we need to generate
# a include file version of the driver file and of
# the pattern files.
# Driver file is similar in this case but different from semihosting
# one.
if not self._fpga:
with open("%s/TestDrive.h" % header,"w") as driverFile:
driverFile.write("// Empty driver include in semihosting mode")
with open("%s/Patterns.h" % header,"w") as includeFile:
includeFile.write("// Empty pattern include in semihosting mode")
else:
with open("%s/TestDrive.h" % header,"w") as driverFile:
driverFile.write("#ifndef _DRIVER_H_\n")
driverFile.write("#define _DRIVER_H_\n")
driverFile.write("__ALIGNED(8) const char testDesc[]={\n")
self._offset=0
with open("%s/Patterns.h" % header,"w") as includeFile:
includeFile.write("#ifndef _PATTERNS_H_\n")
includeFile.write("#define _PATTERNS_H_\n")
includeFile.write("__ALIGNED(8) const char patterns[]={\n")
self._genDriver(root,driverFile,includeFile)
includeFile.write("};\n")
includeFile.write("#endif\n")
driverFile.write("};\n")
driverFile.write("#endif\n")