Skip to content

Modify test framework to derive everything from compiler object #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+123 −367
Open

Modify test framework to derive everything from compiler object #16

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
adb06b3
mild improvements to testing framework
AndrewTolmach Dec 20, 2022
47e8661
new test framework utils.py
AndrewTolmach Dec 21, 2022
d5651bc
typo
AndrewTolmach Dec 21, 2022
8bc8f89
update top level driver and add a new one
AndrewTolmach Dec 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions run-some-tests.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
import os
import sys
import getopt

from utils import run_selected_tests, enable_tracing, enable_emulation

import var

sys.setrecursionlimit(10000)

try:
opts, files = getopt.getopt(sys.argv[1:],"te")
except getopt.GetoptError:
print ('usage: run-tests.py [-t] [-e]')
sys.exit(1)

for opt,arg in opts:
if opt == '-t':
enable_tracing()
if opt == '-e':
enable_emulation()

run_selected_tests(files, compiler.Compiler())

28 changes: 2 additions & 26 deletions run-tests.py
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,11 @@
import os
import compiler
import interp_Lvar
import type_check_Lvar
from utils import run_tests, run_one_test
from interp_x86.eval_x86 import interp_x86

compiler = compiler.Compiler()

typecheck_Lvar = type_check_Lvar.TypeCheckLvar().type_check

typecheck_dict = {
'source': typecheck_Lvar,
'remove_complex_operands': typecheck_Lvar,
}
interpLvar = interp_Lvar.InterpLvar().interp
interp_dict = {
'remove_complex_operands': interpLvar,
'select_instructions': interp_x86,
'assign_homes': interp_x86,
'patch_instructions': interp_x86,
}

if False:
run_one_test(os.getcwd() + '/tests/var/zero.py',
'var',
compiler,
'var',
typecheck_dict,
interp_dict)
compiler)
else:
run_tests('var', compiler, 'var',
typecheck_dict,
interp_dict)

run_tests('var', compiler)
437 changes: 96 additions & 341 deletions utils.py
View file
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
import ast
from ast import *
from dataclasses import dataclass
from contextlib import redirect_stdout,_RedirectStream

# move these to the compilers, use a method with overrides -Jeremy
builtin_functions = \
@@ -766,7 +767,7 @@ class Begin(expr):
def __str__(self):
indent()
stmts = ''.join([str(s) for s in self.body])
end = indent_stmt() + + 'produce ' + str(self.result)
end = indent_stmt() + 'produce ' + str(self.result)
dedent()
return '{\n' + stmts + end + '}'

@@ -1120,24 +1121,38 @@ def label_name(n: str) -> str:

tracing = False

emulate_x86 = False

def enable_tracing():
global tracing
tracing = True

def enable_emulation():
global emulate_x86
emulate_x86 = True


def trace(msg):
if tracing:
print(msg, file=sys.stderr)


def trace_ast_and_concrete(ast):
trace("concrete syntax:")
trace(ast)
trace("")
trace("AST:")
trace(repr(ast))

def is_python_extension(filename):
s = os.path.splitext(filename)
if len(s) > 1:
return s[1] == ".py"
else:
return False

class redirect_stdin(_RedirectStream):
_stream = "stdin"

# Given the `ast` output of a pass and a test program (root) name,
# runs the interpreter on the program and compares the output to the
@@ -1146,14 +1161,12 @@ def test_pass(passname, interp_dict, program_root, ast,
compiler_name):
if passname in interp_dict.keys():
input_file = program_root + '.in'
if not os.path.isfile(input_file):
input_file = "/dev/null"
output_file = program_root + '.out'
stdin = sys.stdin
stdout = sys.stdout
sys.stdin = open(input_file, 'r')
sys.stdout = open(output_file, 'w')
interp_dict[passname](ast)
sys.stdin = stdin
sys.stdout = stdout
with open(input_file, 'r') as inf, open(output_file, 'w') as outf:
with redirect_stdout(outf), redirect_stdin(inf):
interp_dict[passname](ast)
result = os.system('diff' + ' -b ' + output_file + ' ' + program_root + '.golden')
if result == 0:
trace('compiler ' + compiler_name + ' success on pass ' + passname \
@@ -1168,9 +1181,7 @@ def test_pass(passname, interp_dict, program_root, ast,
return 0 # ??


def compile_and_test(compiler, compiler_name,
type_check_dict, interp_dict,
program_filename):
def compile_and_test(compiler, program_filename):
total_passes = 0
successful_passes = 0
from eval_x86 import interp_x86
@@ -1183,226 +1194,23 @@ def compile_and_test(compiler, compiler_name,
trace(program)
trace('')

if 'source' in type_check_dict.keys():
if 'source' in compiler.typecheck_dict.keys():
trace('\n# type checking source program\n')
type_check_dict['source'](program)

passname = 'shrink'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.shrink(program)
trace(program)
trace('')
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program, compiler_name)
else:
trace("\n# no shrink pass!")

passname = 'uniquify'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.uniquify(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program, compiler_name)

passname = 'reveal_functions'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.reveal_functions(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'resolve'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.resolve(program)
trace(program)
trace('')
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program, compiler_name)
compiler.typecheck_dict['source'](program)

passname = 'erase_types'
if hasattr(compiler, passname):
passes = compiler.passes()
for passname, passfn in passes:
trace('\n# ' + passname + '\n')
program = compiler.erase_types(program)
program = passfn(program)
trace(program)
trace('')
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program, compiler_name)

passname = 'cast_insert'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.cast_insert(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'lower_casts'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.lower_casts(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'differentiate_proxies'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.differentiate_proxies(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'reveal_casts'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.reveal_casts(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'convert_assignments'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.convert_assignments(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'convert_to_closures'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.convert_to_closures(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'limit_functions'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.limit_functions(program)
trace(program)
if passname in type_check_dict.keys():
trace('type checking after ' + passname + '\n')
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'expose_allocation'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.expose_allocation(program)
trace(program)
if passname in type_check_dict.keys():
trace('type checking after ' + passname + '\n')
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'remove_complex_operands'
trace('\n# ' + passname + '\n')
program = compiler.remove_complex_operands(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'explicate_control'
if hasattr(compiler, passname):
trace('\n# ' + passname + '\n')
program = compiler.explicate_control(program)
trace(program)
if passname in type_check_dict.keys():
type_check_dict[passname](program)
trace('type checking passed')
else:
trace('skipped type checking')
if passname in compiler.typecheck_dict.keys():
trace('compiler ' + compiler.compiler_name + ' type checking on pass ' + passname \
+ ' on test ' + program_root + '\n')
compiler.typecheck_dict[passname](program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'select_instructions'
trace('\n# ' + passname + '\n')
program = compiler.select_instructions(program)
trace(program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'assign_homes'
trace('\n# ' + passname + '\n')
program = compiler.assign_homes(program)
trace(program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

passname = 'patch_instructions'
trace('\n# ' + passname + '\n')
program = compiler.patch_instructions(program)
trace(program)
total_passes += 1
successful_passes += \
test_pass(passname, interp_dict, program_root, program,
compiler_name)

trace('\n# prelude and conclusion\n')
program = compiler.prelude_and_conclusion(program)
trace(program)
trace("")
test_pass(passname, compiler.interp_dict, program_root, program, compiler.compiler_name)

x86_filename = program_root + ".s"
with open(x86_filename, "w") as dest:
@@ -1411,148 +1219,72 @@ def compile_and_test(compiler, compiler_name,
total_passes += 1

# Run the final x86 program
emulate_x86 = False
input_file = program_root + '.in'
if not os.path.isfile(input_file):
input_file = "/dev/null"
output_file = program_root + '.out'
if emulate_x86:
stdin = sys.stdin
stdout = sys.stdout
sys.stdin = open(program_root + '.in', 'r')
sys.stdout = open(program_root + '.out', 'w')
interp_x86(program)
sys.stdin = stdin
sys.stdout = stdout
trace('emulating x86')
with open(input_file, 'r') as inf, open(output_file, 'w') as outf:
with redirect_stdout(outf), redirect_stdin(inf):
interp_x86(program)
else:
if platform == 'darwin':
os.system('gcc -arch x86_64 runtime.o ' + x86_filename)
else:
os.system('gcc runtime.o ' + x86_filename)
input_file = program_root + '.in'
output_file = program_root + '.out'
os.system('./a.out < ' + input_file + ' > ' + output_file)
trace('executing x86')
arch = ' -arch x86_64' if platform == 'darwin' else ''
x86_executable = program_root + '.exe'
if os.path.isfile(x86_executable):
os.remove(x86_executable)
if os.path.isfile(output_file):
os.remove(output_file)
os.system('gcc' + arch + ' -o ' + x86_executable + ' runtime.o ' + x86_filename)
os.system(x86_executable + ' < ' + input_file + ' > ' + output_file)

result = os.system('diff' + ' -b ' + program_root + '.out ' \
+ program_root + '.golden')
if result == 0:
successful_passes += 1
return (successful_passes, total_passes, 1)
else:
print('compiler ' + compiler_name + ', executable failed' \
print('compiler ' + compiler.compiler_name + ', executable failed' \
+ ' on test ' + program_root)
return (successful_passes, total_passes, 0)


def trace_ast_and_concrete(ast):
trace("concrete syntax:")
trace(ast)
trace("")
trace("AST:")
trace(repr(ast))


# This function compiles the program without any testing
def compile(compiler, compiler_name, type_check_L, type_check_C,
program_filename):
def compile(compiler, program_filename):
program_root = os.path.splitext(program_filename)[0]
with open(program_filename) as source:
program = parse(source.read())

trace('\n# type check\n')
type_check_L(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'shrink'):
trace('\n# shrink\n')
program = compiler.shrink(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'uniquify'):
trace('\n# uniquify\n')
program = compiler.uniquify(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'reveal_functions'):
trace('\n# reveal functions\n')
type_check_L(program)
program = compiler.reveal_functions(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'convert_assignments'):
trace('\n# assignment conversion\n')
type_check_L(program)
program = compiler.convert_assignments(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'limit_functions'):
trace('\n# limit functions\n')
type_check_L(program)
program = compiler.limit_functions(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'convert_to_closures'):
trace('\n# closure conversion\n')
type_check_L(program)
program = compiler.convert_to_closures(program)
trace_ast_and_concrete(program)

if hasattr(compiler, 'expose_allocation'):
trace('\n# expose allocation\n')
type_check_L(program)
program = compiler.expose_allocation(program)
trace_ast_and_concrete(program)

trace('\n# remove complex\n')
program = compiler.remove_complex_operands(program)
trace_ast_and_concrete(program)
trace('\n# type check\n')
compiler.typecheck_dict['source'](program)

if hasattr(compiler, 'explicate_control'):
trace('\n# explicate control\n')
program = compiler.explicate_control(program)
for passname, passfn in compiler.passes():
trace('\n# ' + passname + '\n')
perogram = passfn(program)
trace_ast_and_concrete(program)

if type_check_C:
type_check_C(program)

trace('\n# select instructions\n')
pseudo_x86 = compiler.select_instructions(program)
trace_ast_and_concrete(pseudo_x86)

trace('\n# assign homes\n')
almost_x86 = compiler.assign_homes(pseudo_x86)
trace_ast_and_concrete(almost_x86)

trace('\n# patch instructions\n')
x86 = compiler.patch_instructions(almost_x86)
trace_ast_and_concrete(x86)

trace('\n# prelude and conclusion\n')
x86 = compiler.prelude_and_conclusion(x86)
trace_ast_and_concrete(x86)

# Output x86 program to the .s file
x86_filename = program_root + ".s"
with open(x86_filename, "w") as dest:
dest.write(str(x86))

# Given a test file name, the name of a language, a compiler, a type

dest.write(str(program))

# checker and interpreter for the language, and an interpeter for the
# C intermediate language, run all the passes in the compiler,
# Given a test file name and a compiler, run all the passes in the compiler,
# checking that the resulting programs produce output that matches the
# golden file.
def run_one_test(test, lang, compiler, compiler_name,
type_check_dict, interp_dict):
# test_root = os.path.splitext(test)[0]
# test_name = os.path.basename(test_root)
return compile_and_test(compiler, compiler_name, type_check_dict,
interp_dict, test)


# Given the name of a language, a compiler, the compiler's name, a
# type checker and interpreter for the language, and an interpreter
# for the C intermediate language, test the compiler on all the tests
# in the directory of for the given language, i.e., all the
# python files in ./tests/<language>.
def run_tests(lang, compiler, compiler_name, type_check_dict, interp_dict):
def run_one_test(test, compiler):
try:
return compile_and_test(compiler, test)
except Exception as exn:
from traceback import print_exc
print_exc(file=sys.stderr)
return (0,0,0)

# Given the name <language> of a language and a compiler,
# test the compiler on all the tests in ./tests/<language>
def run_tests(lang, compiler):
# Collect all the test programs for this language.
homedir = os.getcwd()
directory = homedir + '/tests/' + lang + '/'
@@ -1570,15 +1302,38 @@ def run_tests(lang, compiler, compiler_name, type_check_dict, interp_dict):
total_tests = 0
for test in tests:
(succ_passes, tot_passes, succ_test) = \
run_one_test(test, lang, compiler, compiler_name,
type_check_dict, interp_dict)
run_one_test(test, compiler)
successful_passes += succ_passes
total_passes += tot_passes
successful_tests += succ_test
total_tests += 1

# Report the pass/fails
print('tests: ' + repr(successful_tests) + '/' + repr(total_tests) \
+ ' for compiler ' + compiler.compiler_name + ' on language ' + lang)
print('passes: ' + repr(successful_passes) + '/' + repr(total_passes) \
+ ' for compiler ' + compiler.compiler_name + ' on language ' + lang)

# Given a list of file names and a compiler,
# test the compiler on all the tests in the list.
def run_selected_tests(tests, compiler):
# Compile and run each test program, comparing output to the golden file.
successful_passes = 0
total_passes = 0
successful_tests = 0
total_tests = 0
for test in tests:
print(test + ':', file=sys.stderr)
(succ_passes, tot_passes, succ_test) = \
run_one_test(test, compiler)
successful_passes += succ_passes
total_passes += tot_passes
successful_tests += succ_test
total_tests += 1

# Report the pass/fails
print('tests: ' + repr(successful_tests) + '/' + repr(total_tests) \
+ ' for compiler ' + compiler_name + ' on language ' + lang)
+ ' for compiler ' + compiler.compiler_name)
print('passes: ' + repr(successful_passes) + '/' + repr(total_passes) \
+ ' for compiler ' + compiler_name + ' on language ' + lang)
+ ' for compiler ' + compiler.compiler_name)