338 lines
12 KiB
Python
Executable File
338 lines
12 KiB
Python
Executable File
# Process the test results
|
|
# Test status (like passed, or failed with error code)
|
|
|
|
import argparse
|
|
import re
|
|
import TestScripts.NewParser as parse
|
|
import TestScripts.CodeGen
|
|
from collections import deque
|
|
import os.path
|
|
import numpy as np
|
|
import pandas as pd
|
|
import statsmodels.api as sm
|
|
import statsmodels.formula.api as smf
|
|
import csv
|
|
import TestScripts.Deprecate as d
|
|
import sqlite3
|
|
import datetime, time
|
|
import re
|
|
|
|
# For sql table creation
|
|
MKSTRFIELD=[]
|
|
MKBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED']
|
|
MKINTFIELD=['ID', 'CYCLES']
|
|
MKDATEFIELD=[]
|
|
MKKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE',"RUN"]
|
|
MKKEYFIELDID={'CATEGORY':'categoryid',
|
|
'NAME':'testnameid',
|
|
'DATE':'testdateid',
|
|
'PLATFORM':'platformid',
|
|
'CORE':'coreid',
|
|
'COMPILER':'compilerid',
|
|
'TYPE':'typeid',
|
|
'RUN':'runid'}
|
|
|
|
# For csv table value extraction
|
|
VALSTRFIELD=['TESTNAME','VERSION']
|
|
VALBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED']
|
|
VALINTFIELD=['ID', 'CYCLES']
|
|
VALDATEFIELD=[]
|
|
# Some of those fields may be created by the parsing of other fields
|
|
VALKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE']
|
|
|
|
def joinit(iterable, delimiter):
|
|
it = iter(iterable)
|
|
yield next(it)
|
|
for x in it:
|
|
yield delimiter
|
|
yield x
|
|
|
|
def tableExists(c,tableName):
|
|
req=(tableName,)
|
|
r=c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?",req)
|
|
return(r.fetchone() != None)
|
|
|
|
def diff(first, second):
|
|
second = set(second)
|
|
return [item for item in first if item not in second]
|
|
|
|
def getColumns(elem,full):
|
|
colsToKeep=[]
|
|
cols = list(full.columns)
|
|
params = list(elem.params.full)
|
|
common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params)
|
|
|
|
for field in common:
|
|
if field in MKSTRFIELD:
|
|
colsToKeep.append(field)
|
|
if field in MKINTFIELD:
|
|
colsToKeep.append(field)
|
|
if field in MKKEYFIELD:
|
|
colsToKeep.append(field)
|
|
if field in MKDATEFIELD:
|
|
colsToKeep.append(field)
|
|
if field in MKBOOLFIELD:
|
|
colsToKeep.append(field)
|
|
return(colsToKeep)
|
|
|
|
def createTableIfMissing(conn,elem,tableName,full):
|
|
if not tableExists(conn,tableName):
|
|
sql = "CREATE TABLE %s (" % tableName
|
|
cols = list(full.columns)
|
|
params = list(elem.params.full)
|
|
common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params)
|
|
|
|
sql += "%sid INTEGER PRIMARY KEY" % (tableName)
|
|
start = ","
|
|
|
|
for field in params:
|
|
sql += " %s\n %s INTEGER" % (start,field)
|
|
start = ","
|
|
|
|
for field in common:
|
|
if field in MKSTRFIELD:
|
|
sql += "%s\n %s TEXT" % (start,field)
|
|
if field in MKINTFIELD:
|
|
sql += "%s\n %s INTEGER" % (start,field)
|
|
if field in MKKEYFIELD:
|
|
sql += "%s\n %s INTEGER" % (start,MKKEYFIELDID[field])
|
|
if field in MKDATEFIELD:
|
|
sql += "%s\n %s TEXT" % (start,field)
|
|
if field in MKBOOLFIELD:
|
|
sql += "%s\n %s INTEGER" % (start,field)
|
|
start = ","
|
|
# Create foreign keys
|
|
sql += "%sFOREIGN KEY(typeid) REFERENCES TYPE(typeid)," % start
|
|
sql += "FOREIGN KEY(categoryid) REFERENCES CATEGORY(categoryid),"
|
|
sql += "FOREIGN KEY(testnameid) REFERENCES TESTNAME(testnameid),"
|
|
sql += "FOREIGN KEY(testdateid) REFERENCES TESTDATE(testdateid),"
|
|
sql += "FOREIGN KEY(platformid) REFERENCES PLATFORM(platformid),"
|
|
sql += "FOREIGN KEY(coreid) REFERENCES CORE(coreid),"
|
|
sql += "FOREIGN KEY(compilerid) REFERENCES COMPILER(compilerid)"
|
|
sql += "FOREIGN KEY(runid) REFERENCES RUN(runid)"
|
|
sql += " )"
|
|
conn.execute(sql)
|
|
|
|
# Find the key or add it in a table
|
|
def findInTable(conn,table,keystr,strv,key):
|
|
#print(sql)
|
|
r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,))
|
|
result=r.fetchone()
|
|
if result != None:
|
|
return(result[0])
|
|
else:
|
|
conn.execute("INSERT INTO %s(%s) VALUES(?)" % (table,keystr),(strv,))
|
|
conn.commit()
|
|
r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,))
|
|
result=r.fetchone()
|
|
if result != None:
|
|
#print(result)
|
|
return(result[0])
|
|
else:
|
|
return(None)
|
|
|
|
def findInCompilerTable(conn,kind,version):
|
|
#print(sql)
|
|
r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=?" , (kind,version))
|
|
result=r.fetchone()
|
|
if result != None:
|
|
return(result[0])
|
|
else:
|
|
fullDate = datetime.datetime.now()
|
|
dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid")
|
|
conn.execute("INSERT INTO COMPILER(compilerkindid,version,testdateid) VALUES(?,?,?)" ,(kind,version,dateid))
|
|
conn.commit()
|
|
r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=? AND testdateid=?" , (kind,version,dateid))
|
|
result=r.fetchone()
|
|
if result != None:
|
|
#print(result)
|
|
return(result[0])
|
|
else:
|
|
return(None)
|
|
|
|
|
|
def addRows(conn,elem,tableName,full,runid=0):
|
|
# List of columns we have in DB which is
|
|
# different from the columns in the table
|
|
compilerid = 0
|
|
platformid = 0
|
|
coreid = 0
|
|
keep = getColumns(elem,full)
|
|
cols = list(full.columns)
|
|
params = list(elem.params.full)
|
|
common = diff(["TYPE"] + cols , ['OLDID'] + params)
|
|
colNameList = []
|
|
for c in params + keep:
|
|
if c in MKKEYFIELD:
|
|
colNameList.append(MKKEYFIELDID[c])
|
|
else:
|
|
colNameList.append(c)
|
|
colNames = "".join(joinit(colNameList,","))
|
|
#print(colNameList)
|
|
#print(colNames)
|
|
#print(full)
|
|
for index, row in full.iterrows():
|
|
sql = "INSERT INTO %s(%s) VALUES(" % (tableName,colNames)
|
|
keys = {}
|
|
|
|
# Get data from columns
|
|
for field in common:
|
|
if field in VALSTRFIELD:
|
|
keys[field]=row[field]
|
|
if field == "NAME":
|
|
name = row[field]
|
|
if field == "TESTNAME":
|
|
testname = row[field]
|
|
if re.match(r'^.*_f64',testname):
|
|
keys["TYPE"] = "f64"
|
|
if re.match(r'^.*_f32',testname):
|
|
keys["TYPE"] = "f32"
|
|
if re.match(r'^.*_f16',testname):
|
|
keys["TYPE"] = "f16"
|
|
if re.match(r'^.*_q31',testname):
|
|
keys["TYPE"] = "q31"
|
|
if re.match(r'^.*_q15',testname):
|
|
keys["TYPE"] = "q15"
|
|
if re.match(r'^.*_q7',testname):
|
|
keys["TYPE"] = "q7"
|
|
|
|
if re.match(r'^.*_s8',testname):
|
|
keys["TYPE"] = "s8"
|
|
if re.match(r'^.*_u8',testname):
|
|
keys["TYPE"] = "u8"
|
|
if re.match(r'^.*_s16',testname):
|
|
keys["TYPE"] = "s16"
|
|
if re.match(r'^.*_u16',testname):
|
|
keys["TYPE"] = "u16"
|
|
if re.match(r'^.*_s32',testname):
|
|
keys["TYPE"] = "s32"
|
|
if re.match(r'^.*_u32',testname):
|
|
keys["TYPE"] = "u32"
|
|
if re.match(r'^.*_s64',testname):
|
|
keys["TYPE"] = "s64"
|
|
if re.match(r'^.*_u64',testname):
|
|
keys["TYPE"] = "u64"
|
|
|
|
if field in VALINTFIELD:
|
|
keys[field]=row[field]
|
|
if field in VALDATEFIELD:
|
|
keys[field]=row[field]
|
|
if field in VALBOOLFIELD:
|
|
keys[field]=row[field]
|
|
|
|
|
|
keys['RUN']=runid
|
|
# Get foreign keys and create missing data
|
|
for field in common:
|
|
if field in VALKEYFIELD:
|
|
if field == "CATEGORY":
|
|
# Remove type extension to get category name so that
|
|
# all types are maped to same category which will
|
|
# help for post processing.
|
|
testField=re.sub(r'^(.*)[:]([^:]+)(F16|F32|F64|Q31|Q15|Q7)$',r'\1',row[field])
|
|
val = findInTable(conn,"CATEGORY","category",testField,"categoryid")
|
|
keys[field]=val
|
|
if field == "NAME":
|
|
val = findInTable(conn,"TESTNAME","name",row[field],"testnameid")
|
|
keys[field]=val
|
|
if field == "DATE":
|
|
val = findInTable(conn,"TESTDATE","date",str(row[field]),"testdateid")
|
|
keys[field]=val
|
|
if field == "CORE":
|
|
val = findInTable(conn,"CORE","coredef",row[field],"coreid")
|
|
keys[field]=val
|
|
coreid = val
|
|
if field == "PLATFORM":
|
|
val = findInTable(conn,"PLATFORM","platform",row[field],"platformid")
|
|
keys[field]=val
|
|
platformid = val
|
|
if field == "TYPE":
|
|
val = findInTable(conn,"TYPE","type",keys["TYPE"],"typeid")
|
|
keys[field]=val
|
|
if field == "COMPILER":
|
|
compilerkind = findInTable(conn,"COMPILERKIND","compiler",row[field],"compilerkindid")
|
|
compiler = findInCompilerTable(conn,compilerkind,keys["VERSION"])
|
|
keys[field]=compiler
|
|
compilerid = compiler
|
|
|
|
# Generate sql command
|
|
start = ""
|
|
for field in params:
|
|
sql += " %s\n %d" % (start,row[field])
|
|
start = ","
|
|
|
|
for field in keep:
|
|
if field in MKSTRFIELD or field in MKDATEFIELD:
|
|
sql += " %s\n \"%s\"" % (start,keys[field])
|
|
elif field in keep:
|
|
sql += " %s\n %d" % (start,keys[field])
|
|
start = ","
|
|
|
|
sql += " )"
|
|
#print(sql)
|
|
conn.execute(sql)
|
|
conn.commit()
|
|
return({'compilerid':compilerid,'platformid':platformid,'coreid':coreid})
|
|
|
|
def addConfig(conn,config,fullDate):
|
|
dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid")
|
|
conn.execute("INSERT INTO CONFIG(compilerid,platformid,coreid,testdateid) VALUES(?,?,?,?)" ,(config['compilerid'],config['platformid'],config['coreid'],dateid))
|
|
conn.commit()
|
|
|
|
def getGroup(a):
|
|
return(re.sub(r'^(.+)(F64|F32|F16|Q31|Q15|Q7|U32|U16|U8|S32|S16|S8)$',r'\1',a))
|
|
|
|
def addOneBenchmark(elem,fullPath,db,group,runid):
|
|
if os.path.isfile(fullPath):
|
|
full=pd.read_csv(fullPath,dtype={'OLDID': str} ,keep_default_na = False)
|
|
fullDate = datetime.datetime.now()
|
|
full['DATE'] = fullDate
|
|
if group:
|
|
tableName = getGroup(group)
|
|
else:
|
|
tableName = getGroup(elem.data["class"])
|
|
conn = sqlite3.connect(db)
|
|
createTableIfMissing(conn,elem,tableName,full)
|
|
config = addRows(conn,elem,tableName,full,runid)
|
|
addConfig(conn,config,fullDate)
|
|
conn.close()
|
|
|
|
|
|
def addToDB(benchmark,dbpath,elem,group,runid):
|
|
if not elem.data["deprecated"]:
|
|
if elem.params:
|
|
benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv")
|
|
print("Processing %s" % benchPath)
|
|
addOneBenchmark(elem,benchPath,dbpath,group,runid)
|
|
|
|
for c in elem.children:
|
|
addToDB(benchmark,dbpath,c,group,runid)
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Generate summary benchmarks')
|
|
|
|
parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Pickle")
|
|
parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
|
|
#parser.add_argument('-e', action='store_true', help="Embedded test")
|
|
parser.add_argument('-o', nargs='?',type = str, default="bench.db", help="Benchmark database")
|
|
parser.add_argument('-r', nargs='?',type = int, default=0, help="Run ID")
|
|
|
|
parser.add_argument('others', nargs=argparse.REMAINDER, help="Suite class")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.f is not None:
|
|
#p = parse.Parser()
|
|
# Parse the test description file
|
|
#root = p.parse(args.f)
|
|
root=parse.loadRoot(args.f)
|
|
d.deprecate(root,args.others)
|
|
if args.others:
|
|
group=args.others[0]
|
|
else:
|
|
group=None
|
|
addToDB(args.b,args.o,root,group,args.r)
|
|
|
|
else:
|
|
parser.print_help() |