From 348a5593472f1d48155292e9fef09ed45277519a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:27:17 +0100 Subject: [PATCH 01/66] Add shebang for python3.6 --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 pyc.py diff --git a/pyc.py b/pyc.py old mode 100644 new mode 100755 index b7e0079b..9d98173a --- a/pyc.py +++ b/pyc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3.6 import argparse import ast import logging From e8743b68f1246f2c3958ee9314e0f514d2d8e62e Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:48:37 +0100 Subject: [PATCH 02/66] Whitespace --- pyc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index 9d98173a..4c886a0a 100755 --- a/pyc.py +++ b/pyc.py @@ -235,10 +235,10 @@ def visit_If(self, node: ast.If): return 'if ({test}) {{\n{body}\n}}'.format( test=test_src, body=body_src) - def visit_Eq(self, node:ast.Eq): + def visit_Eq(self, node: ast.Eq): return '==' - def visit_Gt(self, node:ast.Gt): + def visit_Gt(self, node: ast.Gt): return '>' def visit_Compare(self, node:ast.Compare): From 255503c1c5cbc0a5f10767c163bf8f04d9209b38 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:48:54 +0100 Subject: [PATCH 03/66] Add support for ast.Lt --- pyc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyc.py b/pyc.py index 4c886a0a..0c7d07bd 100755 --- a/pyc.py +++ b/pyc.py @@ -241,6 +241,9 @@ def visit_Eq(self, node: ast.Eq): def visit_Gt(self, node: ast.Gt): return '>' + def visit_Lt(self, node: ast.Lt): + return '>' + def visit_Compare(self, node:ast.Compare): """Return the C representation of comparison tests""" # Find the C type of the left value From 649fb27c1a112815dd4c29afed3a01f9a939d26b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:49:05 +0100 Subject: [PATCH 04/66] Add support for ast.BoolOp --- pyc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyc.py b/pyc.py index 0c7d07bd..690187d3 100755 --- a/pyc.py +++ b/pyc.py @@ -244,6 +244,10 @@ def visit_Gt(self, node: ast.Gt): def visit_Lt(self, node: ast.Lt): return '>' + def visit_BoolOp(self, node: ast.BoolOp): + return ' {} '.format(self.visit(node.op)).join([ + '({})'.format(self.visit(v)) for v in node.values]) + def visit_Compare(self, node:ast.Compare): """Return the C representation of comparison tests""" # Find the C type of the left value From c3e1de9f8cea5a1116d7e409452cfbcadf44d7d0 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:49:12 +0100 Subject: [PATCH 05/66] Add support for ast.And --- pyc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyc.py b/pyc.py index 690187d3..58564cdf 100755 --- a/pyc.py +++ b/pyc.py @@ -248,6 +248,9 @@ def visit_BoolOp(self, node: ast.BoolOp): return ' {} '.format(self.visit(node.op)).join([ '({})'.format(self.visit(v)) for v in node.values]) + def visit_And(self, node: ast.And): + return '&&' + def visit_Compare(self, node:ast.Compare): """Return the C representation of comparison tests""" # Find the C type of the left value From b9f0ad124f3b3dc5f0337d89e08fdb682365ea5e Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:49:21 +0100 Subject: [PATCH 06/66] Add support for ast.Or --- pyc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyc.py b/pyc.py index 58564cdf..feaa74dc 100755 --- a/pyc.py +++ b/pyc.py @@ -251,6 +251,9 @@ def visit_BoolOp(self, node: ast.BoolOp): def visit_And(self, node: ast.And): return '&&' + def visit_Or(self, node: ast.Or): + return '||' + def visit_Compare(self, node:ast.Compare): """Return the C representation of comparison tests""" # Find the C type of the left value From 49a49f7925da19a63439d70643bc7b8c60575880 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 14:53:06 +0100 Subject: [PATCH 07/66] Create README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..22991c45 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# megadrive-python +A minimalistic Python compiler for the Sega Megadrive aiming to support a narrow subset of Python 3.6. + From 433088a79781a6a97b1d5ebf2b59ecd14617023f Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 15:53:30 +0100 Subject: [PATCH 08/66] Remove magic string Use BUILTIN_TYPES in favor of hardcoded magic string. --- a.out | Bin 0 -> 8716 bytes pyc.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100755 a.out diff --git a/a.out b/a.out new file mode 100755 index 0000000000000000000000000000000000000000..bdc9ff24b1a0fd98f6032ba463bdf9742c653943 GIT binary patch literal 8716 zcmeHNU1%It6uz6ZHLcN21u8ASrr6ZAJm6DC`8%NKWnW9k*?o2b7wm{vAzrUz?pN;x#yla z^PL&Ed(VFJ*Y&#(2;u7$;!KYa!VebPg*Yn$9Er!lCRobZiC2xcjMryq?IvHh`mY+} z`~XKO!^Vklw~MuYpxY*tj@+1z#q^M}oH$kH$R=;R(TDLcn0i?yMSYv&Ou0>*ajR06 z%9UuTWAes(7l1I-Bg(_kodta=-VSB3cqW6Tk?yQ%Efdp zsT{oV&Z&4i)jV}Kb;ZVnl$G3KIv1De+bx;?PM?fSJcvZ=8Sh)}@2*uY4v~hsM%7o`Ma~ zVc{-|pL2BDKnL)=TsP}^&v~CQFz5J;J%_$S>b`sT#{K7(O65d0cqE8C(or~po(A)N zUj}Uao>e;W(WDX32xtT}0vZ90fJQ(gAjEOA`m?!wEnwCwH_Y1h^FI7MtO>JfY&MM> zvT57||C((Yw<1mBw{+9EoogDu!*&P!$Nr}Ar!eb>U%;x?Cg5D&6#f-%TU#&TQZ-sN zqh;0}N?K8OnziGb=D{mfW8Ht2d(4`#ZZ2O6oV?gtg*NZs7wY6QYSvF*MLy5F^SPR8 z8rKp{<9Y_Ufp6xKTL8H&A~%e07m*w6cjCxx7js(`t+#ojsdck<_$Q3a+ADSz-&c(v zt5a)EB5O!w6p1{;L_k6Rz7-5R`3!tv{7tp>_1$(Gz5^#Owue7~p*?)gx_*l5(fZyx z#Kc^C_%viAg^SqurQL3)xQ*RdJS_R(XFfJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)| z1pXfg>>e7NW2U>uC0o~#*n*6u7G$y#DaOUIqG*h>X=gRbWfXm5G_@P_FAjQE%tz80 zOcevL7tpXS&r>@94gKp_wqP&A;e|v$u~%kOGI1&qtCSPyl}8vhIQ!<*Ovpau5HIqq zLrn8TL-b5eqdOW3&4jHt>OT*W|i3V5rAur$@ga;gtDc$*B z0Q!fRw>pg0{2?{Zej@#kfFU`wD2%IHPkQvcM_=^lXhK?jU-oU;N9AD`XxwcN2 literal 0 HcmV?d00001 diff --git a/pyc.py b/pyc.py index feaa74dc..f3162f42 100755 --- a/pyc.py +++ b/pyc.py @@ -148,7 +148,7 @@ def compile(self): .format(self.node.name), self.node) else: # Modules always return int32 - ret_type = 'int32_t' + ret_type = BUILTIN_TYPES['int'] # Convert the arg specifications if type(self.node) == ast.Module: From d2c0f478573f08207d4bf8030f40abfd496f685d Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 15:53:44 +0100 Subject: [PATCH 09/66] Implement string comparison --- pyc.py | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/pyc.py b/pyc.py index f3162f42..be15d703 100755 --- a/pyc.py +++ b/pyc.py @@ -275,13 +275,42 @@ def visit_Compare(self, node:ast.Compare): if left_type != right_type: raise CompileError('mismatched types in comparison', node.left) - parts = [] - parts.append(self.visit(node.left)) - for op, cmp in zip(node.ops, node.comparators): - parts.append(self.visit(op)) - parts.append(self.visit(cmp)) - LOG.debug('compare parts: %r', parts) - return ' '.join(parts) + # String comparison is a different matter in C + if left_type == BUILTIN_TYPES['str']: + # Ensure exactly two items are being compared (left + 1 comparator) + if len(node.comparators) != 1: + raise CompileError( + 'string comparisons must be of exactly two items', node) + + # Determine which strcmp() return value to expect + op = node.ops[0] + comp = '==' + if type(op) == ast.Eq: + expect = 0 + elif type(op) == ast.NotEq: + expect = 0 + comp = '!=' + elif type(op) == ast.Lt: + expect = -1 + elif type(op) == ast.Gt: + expect = 1 + else: + raise CompileError( + 'unsupported string comparison op: {}' + .format(ast.dump(op)), node) + + return 'strcmp({}, {}) {} {}'.format( + self.visit(node.left), self.visit(node.comparators[0]), comp, + expect) + else: + # For non-string comparisons just translate the symbols + parts = [] + parts.append(self.visit(node.left)) + for op, cmp in zip(node.ops, node.comparators): + parts.append(self.visit(op)) + parts.append(self.visit(cmp)) + LOG.debug('compare parts: %r', parts) + return ' '.join(parts) def visit_Call(self, node:ast.Call): # Find the function being called From ac859dba3ca7c7e07f0cbc49722dde2e7b25167a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 16:40:41 +0100 Subject: [PATCH 10/66] #include from libmd SGDK includes a partial implementation of string.h, so this is okay. Normally inclusion of libc code is not okay in this project, but this isn't actually including from libc but rather libmd. --- pyc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyc.py b/pyc.py index be15d703..29d66a3e 100755 --- a/pyc.py +++ b/pyc.py @@ -465,6 +465,7 @@ def __init__(self, module_name, source_filename, node, dunder_name): def _initial_module_source(self): return '\n'.join([ '#include ', + '#include ', '#define {prefix}{mod_name}{DOT}__name__ "{dunder_name}"'.format( prefix=MOD_PREFIX, mod_name=self.module_name, DOT=DOT, dunder_name=self.__name__), From 19e7d90221efbeb50226a76c8f42b0ddc1c250f4 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 16:43:28 +0100 Subject: [PATCH 11/66] Ensure return types match fn's Functions were permitted to be annotated with one return type, but return another. This should not be allowed. It might be desirable in the future to allow return of None from all functions, giving a sort of Optional type behavior, but that would require changing the return type to something like a pointer which can communicate the lack of a value. --- pyc.py | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/pyc.py b/pyc.py index 29d66a3e..b5480c2d 100755 --- a/pyc.py +++ b/pyc.py @@ -49,41 +49,55 @@ def __init__(self, module_name, module_compiler, node): self.locals = {} def _ctype(self, node): - """Distill an ast node down the C type which it will evaluate to + """Distill an ast node down the C type which it will evaluate to. + """ + pytype = self._pytype(node) + return BUILTIN_TYPES[pytype] + + def _pytype(self, node): + """ + Distill an AST node down to the Python type it will evaluate/return. This is actually quite tricky, as a node can be a function call, a reference to a local or global variable, a constant, etc. """ - LOG.debug('determining type of %s', ast.dump(node)) + # If the node is None, the type is None + if node == None: + return None + + # Python modules will always return an int when called + if type(node) == ast.Module: + return 'int' + # For function calls, use the function definition of the callee if type(node) == ast.Call: - return self._ctype(node.func) + return self._pytype(node.func) # For function definitions use the return type annotated if type(node) == ast.FunctionDef: - return self._ctype(node.returns) + return self._pytype(node.returns) # If a variable was declared with an annotated assignment return the # type annotated at the time of assignment if type(node) == ast.AnnAssign: - return self._ctype(node.annotation) + return self._pytype(node.annotation) # If a const value is passed in return the C type for the python type if type(node) == ast.Num: - return BUILTIN_TYPES['int'] + return 'int' if type(node) == ast.Str: - return BUILTIN_TYPES['str'] + return 'str' # If the node passed in is a reference if type(node) == ast.Name: # If the reference is to a builtin type, return that type if node.id in BUILTIN_TYPES: - return BUILTIN_TYPES[node.id] + return node.id # Load the declaration for the variable referenced and return the # storage type annotated at the time of declaration _, val = self._load_name(node) - return self._ctype(val) + return self._pytype(val) # Look for references to None if type(node) == ast.NameConstant and node.value == None: @@ -91,7 +105,7 @@ def _ctype(self, node): # Nothing was found (probably a bug) raise LookupError( - 'BUG: cannot determine C type for {}'.format(ast.dump(node))) + 'BUG: cannot determine python type for {}'.format(ast.dump(node))) def _fn_ret_ctype(self, fn: ast.FunctionDef): # It is okay for functions to lack annotations for return types, but @@ -354,6 +368,14 @@ def visit_Num(self, node: ast.Num): def visit_Return(self, node: ast.Return): """Return the C representation of a python 'return' statement""" + # Ensure the value being returned matches the annotated type of this + # function. + l_type = self._pytype(self.node) + r_type = self._pytype(node.value) + if l_type != r_type: + raise CompileError( + 'cannot return type `{}` from function with return type `{}`' + .format(l_type, r_type), node) return 'return {};'.format(self.visit(node.value)) def visit_Expr(self, node: ast.Expr): From a7f193091ba7c66d93cb9e37efa773442c0d4061 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 17:12:47 +0100 Subject: [PATCH 12/66] Update comment --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index b5480c2d..3180820b 100755 --- a/pyc.py +++ b/pyc.py @@ -560,7 +560,7 @@ def main(): # Add the generated C code to the project source src += compiler.compile() - # Add a main() fn + # Add a main() fn, calling the first module given on the CLI src += 'int main() {{return {}{}{}();}}\n'.format( MOD_PREFIX, args.input_modules[0], MOD_INIT_SUFFIX) From 61f4de6d54395e6891c6095c87fa31ea0686e63c Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 17:14:27 +0100 Subject: [PATCH 13/66] docstring update --- pyc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index 3180820b..46618ce8 100755 --- a/pyc.py +++ b/pyc.py @@ -49,8 +49,7 @@ def __init__(self, module_name, module_compiler, node): self.locals = {} def _ctype(self, node): - """Distill an ast node down the C type which it will evaluate to. - """ + """Distill an ast node down the C type which it will evaluate to.""" pytype = self._pytype(node) return BUILTIN_TYPES[pytype] From bbac9b8e8b4f32e66dbe3d91d326f4e6433ac303 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 17:15:09 +0100 Subject: [PATCH 14/66] Update comment --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 46618ce8..d6c6552b 100755 --- a/pyc.py +++ b/pyc.py @@ -81,7 +81,7 @@ def _pytype(self, node): if type(node) == ast.AnnAssign: return self._pytype(node.annotation) - # If a const value is passed in return the C type for the python type + # If a const value is passed in return the python type if type(node) == ast.Num: return 'int' if type(node) == ast.Str: From 60da5f6ad371e9268a45b07739a8b7ee0806203c Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 17:29:24 +0100 Subject: [PATCH 15/66] Add comments on tricky parts --- pyc.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index d6c6552b..fe64c206 100755 --- a/pyc.py +++ b/pyc.py @@ -163,11 +163,15 @@ def compile(self): # Modules always return int32 ret_type = BUILTIN_TYPES['int'] - # Convert the arg specifications + # Convert the arg specifications for this function/module into a C + # function signature. Modules are just code blocks, so they too are + # implemented as functions in C. if type(self.node) == ast.Module: # Modules have no parameters args_src = '' else: + # Convert each argument individually to C source, and then join + # them all together into a function signature. c_args = [] for arg in self.node.args.args: arg_name = arg.arg @@ -178,16 +182,24 @@ def compile(self): 'missing type annotation for parameter `{}`' .format(arg_name), arg) + # Using the annotation, sort out which C type should be used + # for this argument. try: arg_ctype = self._ctype(arg.annotation) except LookupError: raise CompileError( 'unknown type `{}` for argument `{}`' .format(ast.dump(arg.annotation), arg_name), arg) + + # Add the C source for this argument to a list, which is later + # used to build the function signature. c_args.append('{} {}'.format(arg_ctype, arg_name)) + # Join the individual C source for each argument into one string args_src = ', '.join(c_args) + # Sort out what the name of this C functon will be. Modules need a + # special format, but regular functions are a bit simpler. if type(self.node) == ast.Module: fn_name = MOD_PREFIX + self.module_name + MOD_INIT_SUFFIX else: @@ -201,11 +213,22 @@ def compile(self): ) def generic_visit(self, node): + # This function is called by ast.visit() if there is no such visit_XXX + # function matching the type of the node in the AST tree being visited. + # In other words, this compiler has no function implemented to handle + # the node being passed in. There's nothing to do here except raise an + # error about how there is a missing function in the compiler. LOG.error('Encounteder unsupported node: %r', node) raise CompileError( 'Unsupported ast node: {}'.format(ast.dump(node)), node) def _load_name(self, node): + """Returns the AST node in which a variable was undeclared + + Arguments: + node - an ast.Name or other reference to a variable + + """ # Functions are looked up by name, while others are looked up by id if type(node) == ast.FunctionDef: lookup_name = node.name @@ -231,20 +254,28 @@ def _load_name(self, node): def visit_If(self, node: ast.If): """Return the C representation of a python if statement""" + # Compile the test part of the if block into C code. This is the part + # coming directly after "if", which tests some boolean case. test_src = self.visit(node.test) + # Compile the body to C source. This is the part executed when the test + # evaluates to True. body_src = '' for body_node in node.body: body_src += self.visit(body_node) + # Compile the orelse to C source. This is the part executed when the + # test evaluates to False. if node.orelse: orelse_src = '' for orelse_node in node.orelse: orelse_src += self.visit(orelse_node) + # Return all compiled source in an if...else... format return 'if ({test}) {{\n{body}\n}} else {{\n{orelse}\n}}'.format( test=test_src, body=body_src, orelse=orelse_src) else: + # If there wasn't an else block, return a simpler format return 'if ({test}) {{\n{body}\n}}'.format( test=test_src, body=body_src) From 6e68b9bca928234bcaf05b27712c272a72d06257 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 20:20:18 +0100 Subject: [PATCH 16/66] remove a.out --- a.out | Bin 8716 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 a.out diff --git a/a.out b/a.out deleted file mode 100755 index bdc9ff24b1a0fd98f6032ba463bdf9742c653943..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8716 zcmeHNU1%It6uz6ZHLcN21u8ASrr6ZAJm6DC`8%NKWnW9k*?o2b7wm{vAzrUz?pN;x#yla z^PL&Ed(VFJ*Y&#(2;u7$;!KYa!VebPg*Yn$9Er!lCRobZiC2xcjMryq?IvHh`mY+} z`~XKO!^Vklw~MuYpxY*tj@+1z#q^M}oH$kH$R=;R(TDLcn0i?yMSYv&Ou0>*ajR06 z%9UuTWAes(7l1I-Bg(_kodta=-VSB3cqW6Tk?yQ%Efdp zsT{oV&Z&4i)jV}Kb;ZVnl$G3KIv1De+bx;?PM?fSJcvZ=8Sh)}@2*uY4v~hsM%7o`Ma~ zVc{-|pL2BDKnL)=TsP}^&v~CQFz5J;J%_$S>b`sT#{K7(O65d0cqE8C(or~po(A)N zUj}Uao>e;W(WDX32xtT}0vZ90fJQ(gAjEOA`m?!wEnwCwH_Y1h^FI7MtO>JfY&MM> zvT57||C((Yw<1mBw{+9EoogDu!*&P!$Nr}Ar!eb>U%;x?Cg5D&6#f-%TU#&TQZ-sN zqh;0}N?K8OnziGb=D{mfW8Ht2d(4`#ZZ2O6oV?gtg*NZs7wY6QYSvF*MLy5F^SPR8 z8rKp{<9Y_Ufp6xKTL8H&A~%e07m*w6cjCxx7js(`t+#ojsdck<_$Q3a+ADSz-&c(v zt5a)EB5O!w6p1{;L_k6Rz7-5R`3!tv{7tp>_1$(Gz5^#Owue7~p*?)gx_*l5(fZyx z#Kc^C_%viAg^SqurQL3)xQ*RdJS_R(XFfJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)| z1pXfg>>e7NW2U>uC0o~#*n*6u7G$y#DaOUIqG*h>X=gRbWfXm5G_@P_FAjQE%tz80 zOcevL7tpXS&r>@94gKp_wqP&A;e|v$u~%kOGI1&qtCSPyl}8vhIQ!<*Ovpau5HIqq zLrn8TL-b5eqdOW3&4jHt>OT*W|i3V5rAur$@ga;gtDc$*B z0Q!fRw>pg0{2?{Zej@#kfFU`wD2%IHPkQvcM_=^lXhK?jU-oU;N9AD`XxwcN2 From eaded116773a97b63020dc12a13d1484fb8e61e3 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 5 Dec 2017 20:20:32 +0100 Subject: [PATCH 17/66] add .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..cba7efc8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +a.out From fbc760dcfcc67433a1d899b870201d6fc10d0a25 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 00:37:28 +0100 Subject: [PATCH 18/66] Support module imports This needs work, but successfully adds code from other modules. In order to get this working Attribute lookups need to be completed as well as sorting out a mechanism for initializing imported modules on program start. --- pyc.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index fe64c206..00603e3f 100755 --- a/pyc.py +++ b/pyc.py @@ -529,7 +529,51 @@ def generic_visit(self, node): .format(node), node) def compile(self): - src = self._initial_module_source() + # Store the generated C source code for program in one big string + src = '' + + # Look for import statements + for node in ast.walk(self.node): + if type(node) != ast.Import: + continue + + # Look at what module is being imported + for alias in node.names: + # If the import isn't the form of "import ... as ..." then + # juse use the normal name for the missing "as ..." + asname = alias.asname if alias.asname else alias.name + + # Ensure the asname isn't taken + if asname in self.globals: + raise CompileError( + 'cannot import `{}` multiple times'.format( + asname), node) + + # Look for the module relative to the current working dir + filename = alias.name.replace('.', '/') + '.py' + + # Open the file + with open(filename) as fh: + # Parse the root node using the ast module + root_node = ast.parse(fh.read()) + + # Create a compiler for this module + compiler = ModuleCompiler( + alias.name, filename, root_node, asname) + + # Ensure the module is initialized later + imported_modules.append(asname) + + # Expose that module as a global in this module + self.globals[asname] = compiler + + # Compile the source and add it to the current source string + src += compiler.compile() + + # Add some pre-code for the module + src += self._initial_module_source() + + # Keep track of compilers for the functions in this module func_compilers = [] # Build a compiler for the top-level function @@ -552,6 +596,7 @@ def compile(self): e._msg = '{}:{}'.format(self.source_filename, e._msg) raise + # Return the generated C source code return src @@ -591,6 +636,7 @@ def main(): src += compiler.compile() # Add a main() fn, calling the first module given on the CLI + # TODO: initialize imported functions in main() or somewhere similar src += 'int main() {{return {}{}{}();}}\n'.format( MOD_PREFIX, args.input_modules[0], MOD_INIT_SUFFIX) From 3fac53572376f8f2dfad4614e820a80640f2fa44 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:11:01 +0100 Subject: [PATCH 19/66] Use constants for 'str' and 'int' Use PYTYPE_STR and PYTYPE_INT instead of 'str' and 'int' respectively. Magic strings are bad, as they can cause runtime bugs in cases where constants would cause syntax errors. --- pyc.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pyc.py b/pyc.py index 00603e3f..331ceba5 100755 --- a/pyc.py +++ b/pyc.py @@ -13,10 +13,14 @@ 'print': ast.parse('def printf(s: str): pass').body[0], } +# Define some constants for the names of python types +PYTYPE_INT = 'int' +PYTYPE_STR = 'str' + # These are the C types for various python types supported by this compiler BUILTIN_TYPES = { - 'int': 'int32_t', - 'str': 'char*', + PYTYPE_INT: 'int32_t', + PYTYPE_STR: 'char*', } # A static prefix/suffix for module level things @@ -66,7 +70,7 @@ def _pytype(self, node): # Python modules will always return an int when called if type(node) == ast.Module: - return 'int' + return PYTYPE_INT # For function calls, use the function definition of the callee if type(node) == ast.Call: @@ -83,9 +87,9 @@ def _pytype(self, node): # If a const value is passed in return the python type if type(node) == ast.Num: - return 'int' + return PYTYPE_INT if type(node) == ast.Str: - return 'str' + return PYTYPE_STR # If the node passed in is a reference if type(node) == ast.Name: @@ -161,7 +165,7 @@ def compile(self): .format(self.node.name), self.node) else: # Modules always return int32 - ret_type = BUILTIN_TYPES['int'] + ret_type = BUILTIN_TYPES[PYTYPE_INT] # Convert the arg specifications for this function/module into a C # function signature. Modules are just code blocks, so they too are @@ -320,7 +324,7 @@ def visit_Compare(self, node:ast.Compare): raise CompileError('mismatched types in comparison', node.left) # String comparison is a different matter in C - if left_type == BUILTIN_TYPES['str']: + if left_type == BUILTIN_TYPES[PYTYPE_STR]: # Ensure exactly two items are being compared (left + 1 comparator) if len(node.comparators) != 1: raise CompileError( @@ -448,7 +452,7 @@ def visit_Assign(self, node: ast.Assign): # Handle assingent of numerical constants to variables if type(node.value) == ast.Num: # ensure target is an int - if self.locals[target.id] != 'int': + if self.locals[target.id] != PYTYPE_INT: raise CompileError( 'assignment of int to incompatible {} var {}' .format(self.locals[target.id], target.id), node.value) @@ -465,7 +469,7 @@ def visit_Assign(self, node: ast.Assign): # Handle assignment of string constants to variables elif type(node.value) == ast.Str: # ensure target is a string - if self.locals[target.id] != 'str': + if self.locals[target.id] != PYTYPE_STR: raise CompileError( 'assignment of str to incompatible {} var `{}`' .format(self.locals[target.id], target.id), node.value) @@ -512,7 +516,8 @@ def __init__(self, module_name, source_filename, node, dunder_name): self.__name__ = dunder_name # Declare __name__ as a string global - self.globals['__name__'] = ast.AnnAssign(annotation=ast.Name(id='str')) + self.globals['__name__'] = ast.AnnAssign( + annotation=ast.Name(id=PYTYPE_STR)) def _initial_module_source(self): return '\n'.join([ From 86e48e3085c1e3550a5f2a1be7044ebac048b37f Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:14:29 +0100 Subject: [PATCH 20/66] Use PYTYPE_NONE instead of 'void' This further reduces use of magic strings. --- pyc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index 331ceba5..c595d44f 100755 --- a/pyc.py +++ b/pyc.py @@ -16,11 +16,13 @@ # Define some constants for the names of python types PYTYPE_INT = 'int' PYTYPE_STR = 'str' +PYTYPE_NONE = 'NoneType' # These are the C types for various python types supported by this compiler BUILTIN_TYPES = { PYTYPE_INT: 'int32_t', PYTYPE_STR: 'char*', + PYTYPE_NONE: 'void', } # A static prefix/suffix for module level things @@ -104,7 +106,7 @@ def _pytype(self, node): # Look for references to None if type(node) == ast.NameConstant and node.value == None: - return 'void' + return BUILTIN_TYPES[PYTYPE_NONE] # Nothing was found (probably a bug) raise LookupError( @@ -120,7 +122,7 @@ def _fn_ret_ctype(self, fn: ast.FunctionDef): 'missing return type annotation for function `{}`' .format(self.node.name), self.node) else: - return 'void' + return BUILTIN_TYPES[PYTYPE_NONE] else: # For functions with return type annotations, determine the C type # for the annotated python type From 1e59d82b7c9fead8235e194c7d0e90b497742263 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:22:12 +0100 Subject: [PATCH 21/66] Fix IndentationError --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index c595d44f..6e495bd4 100755 --- a/pyc.py +++ b/pyc.py @@ -454,7 +454,7 @@ def visit_Assign(self, node: ast.Assign): # Handle assingent of numerical constants to variables if type(node.value) == ast.Num: # ensure target is an int - if self.locals[target.id] != PYTYPE_INT: + if self.locals[target.id] != PYTYPE_INT: raise CompileError( 'assignment of int to incompatible {} var {}' .format(self.locals[target.id], target.id), node.value) From f908273d8c4e1b63f03c37976126c76911014838 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:23:44 +0100 Subject: [PATCH 22/66] Do not assume node.id in _load_name When an unknown type is passed into _load_name() raise an error, and do not assume nodes have a .id attribute. --- pyc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index 6e495bd4..5d062918 100755 --- a/pyc.py +++ b/pyc.py @@ -235,11 +235,17 @@ def _load_name(self, node): node - an ast.Name or other reference to a variable """ - # Functions are looked up by name, while others are looked up by id + # Functions are resolved by their name attribute if type(node) == ast.FunctionDef: lookup_name = node.name - else: + # Name nodes are resolved through the id attribute + elif type(node) == ast.Name: lookup_name = node.id + # We don't know how to resolve this node + else: + raise CompileError( + 'unable to resolve reference from node {}'.format( + node), node) # Search in locals first, then module globals, then builtins if lookup_name in self.locals: From 553f0eb88eb472a114e30ad648af90b6925fd216 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:45:07 +0100 Subject: [PATCH 23/66] Improve variable fn_name -> fn_cname This is a better name since the function has two names, and it might be easy to confuse the python name for the C name. --- pyc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyc.py b/pyc.py index 5d062918..b6312418 100755 --- a/pyc.py +++ b/pyc.py @@ -207,15 +207,15 @@ def compile(self): # Sort out what the name of this C functon will be. Modules need a # special format, but regular functions are a bit simpler. if type(self.node) == ast.Module: - fn_name = MOD_PREFIX + self.module_name + MOD_INIT_SUFFIX + fn_cname = MOD_PREFIX + self.module_name + MOD_INIT_SUFFIX else: - fn_name, _ = self._load_name(self.node) + fn_cname, _ = self._load_name(self.node) # TODO: Create pre_src with #include stmts pre_src = '' - return '{pre_src}\n{ret_type} {fn_name}({args}) {{\n{body}}}\n'.format( - pre_src=pre_src, ret_type=ret_type, fn_name=fn_name, args=args_src, - body=src, + return '{pre_src}\n{ret_type} {fn_cname}({args}) {{\n{body}}}\n'.format( + pre_src=pre_src, ret_type=ret_type, fn_cname=fn_cname, + args=args_src, body=src, ) def generic_visit(self, node): From a9ca7a5c6ed3008385416ab775496faecc703afc Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:46:11 +0100 Subject: [PATCH 24/66] Remove TODO about pre_src This has been solved in a different way and now can be removed. --- pyc.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyc.py b/pyc.py index b6312418..33996d0c 100755 --- a/pyc.py +++ b/pyc.py @@ -211,12 +211,8 @@ def compile(self): else: fn_cname, _ = self._load_name(self.node) - # TODO: Create pre_src with #include stmts - pre_src = '' - return '{pre_src}\n{ret_type} {fn_cname}({args}) {{\n{body}}}\n'.format( - pre_src=pre_src, ret_type=ret_type, fn_cname=fn_cname, - args=args_src, body=src, - ) + return '{ret_type} {fn_cname}({args}) {{\n{body}}}\n'.format( + ret_type=ret_type, fn_cname=fn_cname, args=args_src, body=src) def generic_visit(self, node): # This function is called by ast.visit() if there is no such visit_XXX From 795edcdf278d60fcee705593fae6dc80135a1f6c Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:46:40 +0100 Subject: [PATCH 25/66] Fix typo --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 33996d0c..78a74c6b 100755 --- a/pyc.py +++ b/pyc.py @@ -220,7 +220,7 @@ def generic_visit(self, node): # In other words, this compiler has no function implemented to handle # the node being passed in. There's nothing to do here except raise an # error about how there is a missing function in the compiler. - LOG.error('Encounteder unsupported node: %r', node) + LOG.error('Encountered unsupported node: %r', node) raise CompileError( 'Unsupported ast node: {}'.format(ast.dump(node)), node) From d27d715144e5d724e3f37e34536ba3a3f6a1e1d5 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 13:50:04 +0100 Subject: [PATCH 26/66] Move more magic strings to constants --- pyc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyc.py b/pyc.py index 78a74c6b..a38e81f4 100755 --- a/pyc.py +++ b/pyc.py @@ -15,14 +15,17 @@ # Define some constants for the names of python types PYTYPE_INT = 'int' +CTYPE_INT = 'int32_t' PYTYPE_STR = 'str' +CTYPE_STR = 'char*' PYTYPE_NONE = 'NoneType' +CTYPE_NONE = 'void' # These are the C types for various python types supported by this compiler BUILTIN_TYPES = { - PYTYPE_INT: 'int32_t', - PYTYPE_STR: 'char*', - PYTYPE_NONE: 'void', + PYTYPE_INT: CTYPE_INT, + PYTYPE_STR: CTYPE_STR, + PYTYPE_NONE: CTYPE_NONE, } # A static prefix/suffix for module level things From 46df21615cf8dc22094e9bfb86a77fb70705802b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 14:57:31 +0100 Subject: [PATCH 27/66] Add comment --- pyc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyc.py b/pyc.py index a38e81f4..413e6fa5 100755 --- a/pyc.py +++ b/pyc.py @@ -148,6 +148,7 @@ def compile(self): for arg in self.node.args.args: self.locals[arg.arg] = arg + # Generate C source for each AST node under this function src = '' for body_node in self.node.body: node_src = self.visit(body_node) From d702f48e3453b79de81bebc66aa02bd8d0228b9b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 15:08:10 +0100 Subject: [PATCH 28/66] Fix a super obvious bug Whoops...maybe I should build some test cases soon? --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 413e6fa5..a9532ffb 100755 --- a/pyc.py +++ b/pyc.py @@ -298,7 +298,7 @@ def visit_Gt(self, node: ast.Gt): return '>' def visit_Lt(self, node: ast.Lt): - return '>' + return '<' def visit_BoolOp(self, node: ast.BoolOp): return ' {} '.format(self.visit(node.op)).join([ From 29fe76192eb44c9586c1c35b600387afedb21a5a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 15:08:43 +0100 Subject: [PATCH 29/66] Move decorator check earlier Do input check prior to doing work. --- pyc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyc.py b/pyc.py index a9532ffb..866bae22 100755 --- a/pyc.py +++ b/pyc.py @@ -140,6 +140,12 @@ def _fn_ret_ctype(self, fn: ast.FunctionDef): def compile(self): LOG.debug('Compiling: ' + ast.dump(self.node)) + # Check for use of decorators, which is not supported + if type(self.node) == ast.FunctionDef and self.node.decorator_list: + raise CompileError( + 'function decorators are not supported', + self.node.decorator_list[0]) + # Fill the locals with parameters passed into the function if type(self.node) == ast.FunctionDef: # This is confusing for sure...here is an example data structure: @@ -156,12 +162,6 @@ def compile(self): if node_src: src += node_src + '\n' - # Check for use of decorators, which is not supported - if type(self.node) == ast.FunctionDef and self.node.decorator_list: - raise CompileError( - 'function decorators are not supported', - self.node.decorator_list[0]) - # Get the return type for this function if type(self.node) != ast.Module: ret_type = self._fn_ret_ctype(self.node) From 5aa873345cb6680a89a95f778273ff67bb000b41 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:03:06 +0100 Subject: [PATCH 30/66] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cba7efc8..596fecfc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ a.out +test.c From 6d6f5155432d7db74405f199ad2f64db6239780f Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:04:28 +0100 Subject: [PATCH 31/66] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 596fecfc..e5e39150 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ a.out test.c +test.bin From d2840acf48055724b52e884d0a2c5f82750bca17 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:06:16 +0100 Subject: [PATCH 32/66] Add some test cases This is quick and dirty, and could be improved by using nose etc. later. Note: Sadly, these tests currently fail. --- Dockerfile | 2 ++ run_tests.sh | 23 +++++++++++++++++++++++ tests/and/run.sh | 4 ++++ tests/and/test.py | 11 +++++++++++ tests/int/run.sh | 4 ++++ tests/int/test.py | 3 +++ 6 files changed, 47 insertions(+) create mode 100755 run_tests.sh create mode 100644 tests/and/run.sh create mode 100644 tests/and/test.py create mode 100644 tests/int/run.sh create mode 100644 tests/int/test.py diff --git a/Dockerfile b/Dockerfile index d861c44b..959e68bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ FROM python:3.6 COPY . /src +WORKDIR /src +RUN ./run_tests.sh ENTRYPOINT /src/entrypoint.sh diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..71d3cca6 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +export GCC=gcc +export PYC=$(pwd)/pyc.py + +FAILED=0 +PASSED=0 +for test_script in $(find $(pwd)/tests -name run.sh); do + (cd $(dirname $test_script); bash -e $test_script) + if [ $? -ne 0 ]; then + echo "FAILED: $test_script" + let FAILED++ + else + echo "PASSED: $test_script" + let PASSED++ + fi +done + +echo "" +echo "Summary:" +echo " FAILED: $FAILED" +echo " PASSED: $PASSED" + +exit $FAILED diff --git a/tests/and/run.sh b/tests/and/run.sh new file mode 100644 index 00000000..c1ffc148 --- /dev/null +++ b/tests/and/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh +${PYC} test > test.c +${GCC} test.c -otest.bin +./test.bin diff --git a/tests/and/test.py b/tests/and/test.py new file mode 100644 index 00000000..e76b402e --- /dev/null +++ b/tests/and/test.py @@ -0,0 +1,11 @@ +if True and True: + print("OK\n") + +if False and True: + print("ERR\n") + +if False and False: + print("ERR\n") + +if True and False: + print("ERR\n") diff --git a/tests/int/run.sh b/tests/int/run.sh new file mode 100644 index 00000000..c1ffc148 --- /dev/null +++ b/tests/int/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh +${PYC} test > test.c +${GCC} test.c -otest.bin +./test.bin diff --git a/tests/int/test.py b/tests/int/test.py new file mode 100644 index 00000000..a04f655e --- /dev/null +++ b/tests/int/test.py @@ -0,0 +1,3 @@ +x: int = 1 +y: int = 34 +print(x + y) From e5b42e4b9ec3b753bdad1cfd753dfad8752c14b4 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:06:53 +0100 Subject: [PATCH 33/66] Add support for True and False --- pyc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyc.py b/pyc.py index 866bae22..f0f18de4 100755 --- a/pyc.py +++ b/pyc.py @@ -402,6 +402,16 @@ def visit_Name(self, node:ast.Name): cname, _ = self._load_name(node) return cname + def visit_NameConstant(self, node: ast.NameConstant): + """Returns the C name of a python constant""" + if node.value == True: + return 'true' + elif node.value == False: + return 'false' + else: + raise CompileError( + 'could not compile constant `{}`'.format(ast.dump(node)), node) + def visit_Num(self, node: ast.Num): """Return the C representation of a python numerical value""" # Numbers are represented just the same in C as they are in python, so @@ -531,6 +541,7 @@ def _initial_module_source(self): return '\n'.join([ '#include ', '#include ', + '#include ', '#define {prefix}{mod_name}{DOT}__name__ "{dunder_name}"'.format( prefix=MOD_PREFIX, mod_name=self.module_name, DOT=DOT, dunder_name=self.__name__), From d7aac620ec02e9cc5f358a73baa98d342850995c Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:07:10 +0100 Subject: [PATCH 34/66] Add support for '+' (Add) op --- pyc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyc.py b/pyc.py index f0f18de4..d1c4cd0d 100755 --- a/pyc.py +++ b/pyc.py @@ -304,6 +304,9 @@ def visit_BoolOp(self, node: ast.BoolOp): return ' {} '.format(self.visit(node.op)).join([ '({})'.format(self.visit(v)) for v in node.values]) + def visit_Add(self, node: ast.Add): + return '+' + def visit_And(self, node: ast.And): return '&&' From 0e81084f90b1112196c204ab97a7dcf6ede0015d Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:07:36 +0100 Subject: [PATCH 35/66] Add support for BinOp This is incomplete as it only will be valid for numerical types at the moment. --- pyc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyc.py b/pyc.py index d1c4cd0d..d88e91eb 100755 --- a/pyc.py +++ b/pyc.py @@ -304,6 +304,13 @@ def visit_BoolOp(self, node: ast.BoolOp): return ' {} '.format(self.visit(node.op)).join([ '({})'.format(self.visit(v)) for v in node.values]) + def visit_BinOp(self, node: ast.BinOp): + left_src = self.visit(node.left) + right_src = self.visit(node.right) + op_src = self.visit(node.op) + return '{left} {op} {right}'.format( + left=left_src, op=op_src, right=right_src) + def visit_Add(self, node: ast.Add): return '+' From 566410517177e6b980f2d92bd95d224aecd80124 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 16:16:41 +0100 Subject: [PATCH 36/66] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e5e39150..6e390aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ a.out test.c test.bin +.vscode +*.code-workspace From ac628330c9674b2329f0bd478c7602b9a29b7eb9 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 6 Dec 2017 17:14:12 +0100 Subject: [PATCH 37/66] Define exactly what we use in C --- pyc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index d88e91eb..41e9c517 100755 --- a/pyc.py +++ b/pyc.py @@ -550,8 +550,10 @@ def __init__(self, module_name, source_filename, node, dunder_name): def _initial_module_source(self): return '\n'.join([ '#include ', - '#include ', - '#include ', + '#define true 1', + '#define false 0', + 'extern int strcmp(const char*, const char*);', + 'extern int printf(const char*, ...);', '#define {prefix}{mod_name}{DOT}__name__ "{dunder_name}"'.format( prefix=MOD_PREFIX, mod_name=self.module_name, DOT=DOT, dunder_name=self.__name__), From 1f1190d9e05a539d3b6c2bd5cbbfd78a0ad0cb7f Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Thu, 7 Dec 2017 02:42:13 +0100 Subject: [PATCH 38/66] Use purpose built Scope class for locals/globals Using simple dicsts for variable scopes has its limits. This change introduces some regressions in the area of type checking, but that can be improved later. --- pyc.py | 242 +++++++++++++++++++++------------------------------------ 1 file changed, 88 insertions(+), 154 deletions(-) diff --git a/pyc.py b/pyc.py index 41e9c517..13730923 100755 --- a/pyc.py +++ b/pyc.py @@ -14,19 +14,14 @@ } # Define some constants for the names of python types -PYTYPE_INT = 'int' +PYTYPE_INT = int CTYPE_INT = 'int32_t' -PYTYPE_STR = 'str' + +PYTYPE_STR = str CTYPE_STR = 'char*' -PYTYPE_NONE = 'NoneType' -CTYPE_NONE = 'void' -# These are the C types for various python types supported by this compiler -BUILTIN_TYPES = { - PYTYPE_INT: CTYPE_INT, - PYTYPE_STR: CTYPE_STR, - PYTYPE_NONE: CTYPE_NONE, -} +PYTYPE_NONE = type(None) +CTYPE_NONE = 'void' # A static prefix/suffix for module level things MOD_PREFIX = 'PYMOD_' @@ -35,6 +30,7 @@ # How are dots from python references represented in C? DOT = '_DOT_' + class CompileError(RuntimeError): def __init__(self, msg, node): if type(node) != ast.Module: @@ -49,71 +45,60 @@ def __str__(self): return self._msg +class Scope(object): + def __init__(self, parent=None, locals={}): + self.parent = parent + self.locals = locals + + def get(self, pyname): + if pyname in self.locals: + return self.locals[pyname] + if self.parent: + return self.parent.get(pyname) + + def set(self, pyname, pyvalue): + self.locals[pyname] = pyvalue + + def contains(self, key): + if key in self.locals: + return True + if self.parent: + return self.parent.contains(key) + return False + + +class PyType(object): + def __init__(self, pytype, ctype): + self._pytype = pytype + self._ctype = ctype + + def pytype(self): + return self._pytype + + def ctype(self): + return self._ctype + + +IntType = PyType(PYTYPE_INT, CTYPE_INT) +StrType = PyType(PYTYPE_STR, CTYPE_STR) + +class FuncType(PyType): + def __init__(self, args): + self.args = args + +def pytype_from_str(type_name): + if type_name == 'int': + return StrType + if type_name == 'str': + return IntType + raise LookupError() + class FunctionCompiler(ast.NodeVisitor): def __init__(self, module_name, module_compiler, node): self.module_name = module_name self.module_compiler = module_compiler self.node = node - - self.locals = {} - - def _ctype(self, node): - """Distill an ast node down the C type which it will evaluate to.""" - pytype = self._pytype(node) - return BUILTIN_TYPES[pytype] - - def _pytype(self, node): - """ - Distill an AST node down to the Python type it will evaluate/return. - - This is actually quite tricky, as a node can be a function call, - a reference to a local or global variable, a constant, etc. - """ - # If the node is None, the type is None - if node == None: - return None - - # Python modules will always return an int when called - if type(node) == ast.Module: - return PYTYPE_INT - - # For function calls, use the function definition of the callee - if type(node) == ast.Call: - return self._pytype(node.func) - - # For function definitions use the return type annotated - if type(node) == ast.FunctionDef: - return self._pytype(node.returns) - - # If a variable was declared with an annotated assignment return the - # type annotated at the time of assignment - if type(node) == ast.AnnAssign: - return self._pytype(node.annotation) - - # If a const value is passed in return the python type - if type(node) == ast.Num: - return PYTYPE_INT - if type(node) == ast.Str: - return PYTYPE_STR - - # If the node passed in is a reference - if type(node) == ast.Name: - # If the reference is to a builtin type, return that type - if node.id in BUILTIN_TYPES: - return node.id - - # Load the declaration for the variable referenced and return the - # storage type annotated at the time of declaration - _, val = self._load_name(node) - return self._pytype(val) - - # Look for references to None - if type(node) == ast.NameConstant and node.value == None: - return BUILTIN_TYPES[PYTYPE_NONE] - - # Nothing was found (probably a bug) - raise LookupError( - 'BUG: cannot determine python type for {}'.format(ast.dump(node))) + self.scope = Scope(parent=module_compiler.scope) def _fn_ret_ctype(self, fn: ast.FunctionDef): # It is okay for functions to lack annotations for return types, but @@ -146,13 +131,15 @@ def compile(self): 'function decorators are not supported', self.node.decorator_list[0]) - # Fill the locals with parameters passed into the function + # Define a local variable for each argument this function + # will receive at runtime. if type(self.node) == ast.FunctionDef: # This is confusing for sure...here is an example data structure: # FunctionDef(name='main', args=arguments(args=[arg(arg='x', annotation=None), ... # See the docs on ast.FunctionDef, ast.arguments, ast.args, and ast.arg for arg in self.node.args.args: - self.locals[arg.arg] = arg + raise RuntimeError(ast.dump(arg.annotation)) + self.scope.set(arg.arg, arg.annotation) # Generate C source for each AST node under this function src = '' @@ -171,7 +158,7 @@ def compile(self): .format(self.node.name), self.node) else: # Modules always return int32 - ret_type = BUILTIN_TYPES[PYTYPE_INT] + ret_type = CTYPE_INT # Convert the arg specifications for this function/module into a C # function signature. Modules are just code blocks, so they too are @@ -228,41 +215,9 @@ def generic_visit(self, node): raise CompileError( 'Unsupported ast node: {}'.format(ast.dump(node)), node) - def _load_name(self, node): - """Returns the AST node in which a variable was undeclared - - Arguments: - node - an ast.Name or other reference to a variable - - """ - # Functions are resolved by their name attribute - if type(node) == ast.FunctionDef: - lookup_name = node.name - # Name nodes are resolved through the id attribute - elif type(node) == ast.Name: - lookup_name = node.id - # We don't know how to resolve this node - else: - raise CompileError( - 'unable to resolve reference from node {}'.format( - node), node) - - # Search in locals first, then module globals, then builtins - if lookup_name in self.locals: - # Items at the local scope have the same variable name in C. - return [lookup_name, self.locals[lookup_name]] - elif lookup_name in self.module_compiler.globals: - # Items resolving at the module level have a more complex naming - # scheme. - return [ - ''.join([MOD_PREFIX, self.module_name, DOT, lookup_name]), - self.module_compiler.globals[lookup_name] - ] - elif lookup_name in BUILTIN_FUNCS: - # Builtin functions have different names in C from python - return BUILTIN_FUNCS[lookup_name].name, BUILTIN_FUNCS[lookup_name] - else: - raise LookupError('no such var `{}`'.format(lookup_name)) + def visit_Attribute(self, node: ast.Attribute): + """Resolve refrences to attributes of objects (such as a.b.c)""" + return '{}.{}'.format(self.visit(node.value), node.attr) def visit_If(self, node: ast.If): """Return the C representation of a python if statement""" @@ -379,25 +334,12 @@ def visit_Compare(self, node:ast.Compare): return ' '.join(parts) def visit_Call(self, node:ast.Call): - # Find the function being called - try: - func_name, func = self._load_name(node.func) - except LookupError: - raise CompileError( - 'reference to unknown function `{}`'.format(node.func.id), - node) - - # Ensure the function being called is infact a function - if type(func) != ast.FunctionDef: - raise CompileError( - 'call to non-function `{}` of type `{}`' - .format(node.func.id, func), node) - + # TODO: Check that the function exists # TODO: Check arguments cargs = [] for arg in node.args: cargs.append(self.visit(arg)) - return '{}({})'.format(func_name, ', '.join(cargs)) + return '{}({})'.format(node.func.id, ', '.join(cargs)) def visit_Str(self, node:ast.Str): """Return the C representation of a python string""" @@ -409,15 +351,18 @@ def visit_Name(self, node:ast.Name): """Returns the C name of a python variable""" # _load_name returns both the cname and the ast node, but we only need # the name. - cname, _ = self._load_name(node) - return cname + return node.id def visit_NameConstant(self, node: ast.NameConstant): """Returns the C name of a python constant""" + # Check if the node refers to the python constant 'True' if node.value == True: return 'true' + # Check if the node refers to the python constant 'False' elif node.value == False: return 'false' + # There are certainly other constants with no implementation here, so + # just raise an error else: raise CompileError( 'could not compile constant `{}`'.format(ast.dump(node)), node) @@ -432,12 +377,7 @@ def visit_Return(self, node: ast.Return): """Return the C representation of a python 'return' statement""" # Ensure the value being returned matches the annotated type of this # function. - l_type = self._pytype(self.node) - r_type = self._pytype(node.value) - if l_type != r_type: - raise CompileError( - 'cannot return type `{}` from function with return type `{}`' - .format(l_type, r_type), node) + # TODO: Check that the type is valid to return return 'return {};'.format(self.visit(node.value)) def visit_Expr(self, node: ast.Expr): @@ -472,7 +412,7 @@ def visit_Assign(self, node: ast.Assign): 'Use of unsupported feature: attribute assignment', node) # Ensure the variable has been declared - if target.id not in self.locals: + if not self.scope.contains(target.id): raise CompileError( 'Cannot assign to undeclared local var `{}`' .format(target.id), node) @@ -480,16 +420,16 @@ def visit_Assign(self, node: ast.Assign): # Handle assingent of numerical constants to variables if type(node.value) == ast.Num: # ensure target is an int - if self.locals[target.id] != PYTYPE_INT: + if self.scope.get(target.id).pytype() != PYTYPE_INT: raise CompileError( 'assignment of int to incompatible {} var {}' - .format(self.locals[target.id], target.id), node.value) + .format(self.scope.get(target.id), target.id), node.value) # prevent float assignment if '.' in str(node.value.n): raise CompileError( 'assignment of float to incompatible {} var {}' - .format(self.locals[target.id], target.id), node.value) + .format(self.scope.get(target.id), target.id), node.value) # output the code return '{} = {};\n'.format(target.id, node.value.n) @@ -497,10 +437,10 @@ def visit_Assign(self, node: ast.Assign): # Handle assignment of string constants to variables elif type(node.value) == ast.Str: # ensure target is a string - if self.locals[target.id] != PYTYPE_STR: + if self.scope[target.id] != PYTYPE_STR: raise CompileError( 'assignment of str to incompatible {} var `{}`' - .format(self.locals[target.id], target.id), node.value) + .format(self.scope.get(target.id), target.id), node.value) # output the code return '{} = "{}";\n'.format( @@ -508,44 +448,38 @@ def visit_Assign(self, node: ast.Assign): else: raise CompileError( 'Unsupported assignment of type `{}` to `{}` of type `{}`' - .format(type(node.value), target.id, self.locals[target.id]), + .format(type(node.value), target.id, self.scope.get(target.id)), node.value) def visit_AnnAssign(self, node: ast.AnnAssign): # Sort out whether this is a new local declaration - if node.target.id in self.locals: + if self.scope.contains(node.target.id): raise CompilerError( 'Local var `{}` has already been declared' .format(node.target.id)) - # Ensure the data types match for assignment - target_type = self._ctype(node) - value_type = self._ctype(node.value) - if target_type != value_type: - raise CompileError( - 'type mismatch in assignment of {} to {}'.format( - node.value, node.target), node.value) + # TODO: Ensure the data types match for assignment # Store this declaration in the locals table - self.locals[node.target.id] = node + # TODO: Do not assume node.annotation has an `id` attr + self.scope.set(node.target.id, pytype_from_str(node.annotation.id)) # Generate C code for the assignment target_src = self.visit(node.target) value_src = self.visit(node.value) - return '{} {} = {};'.format(target_type, target_src, value_src) + return 'int {} = {};'.format(target_src, value_src) class ModuleCompiler(ast.NodeVisitor): def __init__(self, module_name, source_filename, node, dunder_name): self.module_name = module_name - self.globals = {} + self.scope = Scope() self.node = node self.source_filename = source_filename self.__name__ = dunder_name # Declare __name__ as a string global - self.globals['__name__'] = ast.AnnAssign( - annotation=ast.Name(id=PYTYPE_STR)) + self.scope.set('__name__', StrType) def _initial_module_source(self): return '\n'.join([ @@ -580,7 +514,7 @@ def compile(self): asname = alias.asname if alias.asname else alias.name # Ensure the asname isn't taken - if asname in self.globals: + if self.scope.contains(asname): raise CompileError( 'cannot import `{}` multiple times'.format( asname), node) @@ -601,7 +535,7 @@ def compile(self): imported_modules.append(asname) # Expose that module as a global in this module - self.globals[asname] = compiler + self.scope.set(asname, ModuleType(scope=compiler.scope)) # Compile the source and add it to the current source string src += compiler.compile() @@ -619,7 +553,7 @@ def compile(self): # Build a compiler for all other functions for mod_node in self.node.body: if type(mod_node) == ast.FunctionDef: - self.globals[mod_node.name] = mod_node + self.scope.set(mod_node.name, FuncType(args=mod_name.args)) func_compiler = FunctionCompiler( self.module_name, self, mod_node) func_compilers.append(func_compiler) From 9045fea777ef8bc68a9c75aa18fa6c7630c39d01 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:14:33 +0100 Subject: [PATCH 39/66] Switch to multi-tiered NodeVisitor design Iterate, iterate, iterate! --- pyc.py | 641 ++++++--------------------------------------------------- 1 file changed, 65 insertions(+), 576 deletions(-) diff --git a/pyc.py b/pyc.py index 13730923..15b36921 100755 --- a/pyc.py +++ b/pyc.py @@ -7,610 +7,99 @@ LOG = logging.getLogger(__name__) -# These functions are called in python by name on left and in C name by name -# on right. -BUILTIN_FUNCS = { - 'print': ast.parse('def printf(s: str): pass').body[0], -} +class CompileError(RuntimeError): pass -# Define some constants for the names of python types -PYTYPE_INT = int -CTYPE_INT = 'int32_t' - -PYTYPE_STR = str -CTYPE_STR = 'char*' - -PYTYPE_NONE = type(None) -CTYPE_NONE = 'void' - -# A static prefix/suffix for module level things -MOD_PREFIX = 'PYMOD_' -MOD_INIT_SUFFIX = '_INIT' - -# How are dots from python references represented in C? -DOT = '_DOT_' - - -class CompileError(RuntimeError): - def __init__(self, msg, node): - if type(node) != ast.Module: - self._msg = '{}:{} {}'.format(node.lineno, node.col_offset, msg) - else: - self._msg = msg - - def msg(self): - return 'CompileError: {}'.format(self._msg) - - def __str__(self): - return self._msg - - -class Scope(object): - def __init__(self, parent=None, locals={}): - self.parent = parent - self.locals = locals - - def get(self, pyname): - if pyname in self.locals: - return self.locals[pyname] - if self.parent: - return self.parent.get(pyname) - - def set(self, pyname, pyvalue): - self.locals[pyname] = pyvalue - - def contains(self, key): - if key in self.locals: - return True - if self.parent: - return self.parent.contains(key) - return False - - -class PyType(object): - def __init__(self, pytype, ctype): - self._pytype = pytype - self._ctype = ctype - - def pytype(self): - return self._pytype - - def ctype(self): - return self._ctype - - -IntType = PyType(PYTYPE_INT, CTYPE_INT) -StrType = PyType(PYTYPE_STR, CTYPE_STR) - -class FuncType(PyType): - def __init__(self, args): - self.args = args - -def pytype_from_str(type_name): - if type_name == 'int': - return StrType - if type_name == 'str': - return IntType - raise LookupError() - -class FunctionCompiler(ast.NodeVisitor): - def __init__(self, module_name, module_compiler, node): - self.module_name = module_name - self.module_compiler = module_compiler - self.node = node - self.scope = Scope(parent=module_compiler.scope) - - def _fn_ret_ctype(self, fn: ast.FunctionDef): - # It is okay for functions to lack annotations for return types, but - # only if they do not contain any return statement - if self.node.returns == None: - for fn_node in ast.walk(self.node): - if type(fn_node) == ast.Return: - raise CompileError( - 'missing return type annotation for function `{}`' - .format(self.node.name), self.node) - else: - return BUILTIN_TYPES[PYTYPE_NONE] - else: - # For functions with return type annotations, determine the C type - # for the annotated python type - try: - return self._ctype(self.node.returns) - except LookupError: - raise CompileError( - 'unsupported return type `{}` for function `{}`' - .format(ast.dump(self.node.returns), fn_name), - self.node) - - def compile(self): - LOG.debug('Compiling: ' + ast.dump(self.node)) - - # Check for use of decorators, which is not supported - if type(self.node) == ast.FunctionDef and self.node.decorator_list: - raise CompileError( - 'function decorators are not supported', - self.node.decorator_list[0]) - - # Define a local variable for each argument this function - # will receive at runtime. - if type(self.node) == ast.FunctionDef: - # This is confusing for sure...here is an example data structure: - # FunctionDef(name='main', args=arguments(args=[arg(arg='x', annotation=None), ... - # See the docs on ast.FunctionDef, ast.arguments, ast.args, and ast.arg - for arg in self.node.args.args: - raise RuntimeError(ast.dump(arg.annotation)) - self.scope.set(arg.arg, arg.annotation) - - # Generate C source for each AST node under this function - src = '' - for body_node in self.node.body: - node_src = self.visit(body_node) - LOG.debug('source for node %s: %r', body_node, node_src) - if node_src: - src += node_src + '\n' - - # Get the return type for this function - if type(self.node) != ast.Module: - ret_type = self._fn_ret_ctype(self.node) - if ret_type == None: - raise CompileError( - 'unable to determine return type of function `{}`' - .format(self.node.name), self.node) - else: - # Modules always return int32 - ret_type = CTYPE_INT - - # Convert the arg specifications for this function/module into a C - # function signature. Modules are just code blocks, so they too are - # implemented as functions in C. - if type(self.node) == ast.Module: - # Modules have no parameters - args_src = '' - else: - # Convert each argument individually to C source, and then join - # them all together into a function signature. - c_args = [] - for arg in self.node.args.args: - arg_name = arg.arg - - # Ensure the type was annotated for this argument - if not arg.annotation: - raise CompileError( - 'missing type annotation for parameter `{}`' - .format(arg_name), arg) - - # Using the annotation, sort out which C type should be used - # for this argument. - try: - arg_ctype = self._ctype(arg.annotation) - except LookupError: - raise CompileError( - 'unknown type `{}` for argument `{}`' - .format(ast.dump(arg.annotation), arg_name), arg) - - # Add the C source for this argument to a list, which is later - # used to build the function signature. - c_args.append('{} {}'.format(arg_ctype, arg_name)) - - # Join the individual C source for each argument into one string - args_src = ', '.join(c_args) - - # Sort out what the name of this C functon will be. Modules need a - # special format, but regular functions are a bit simpler. - if type(self.node) == ast.Module: - fn_cname = MOD_PREFIX + self.module_name + MOD_INIT_SUFFIX - else: - fn_cname, _ = self._load_name(self.node) - - return '{ret_type} {fn_cname}({args}) {{\n{body}}}\n'.format( - ret_type=ret_type, fn_cname=fn_cname, args=args_src, body=src) +class BaseCompiler(ast.NodeVisitor): + def __init__(self, name, root): + self.name = name + self.root = root + self.docstring = '' + try: + if type(root.body[0]) == ast.Str: + self.docstring = root.body[0].s + except AttributeError: + pass def generic_visit(self, node): - # This function is called by ast.visit() if there is no such visit_XXX - # function matching the type of the node in the AST tree being visited. - # In other words, this compiler has no function implemented to handle - # the node being passed in. There's nothing to do here except raise an - # error about how there is a missing function in the compiler. - LOG.error('Encountered unsupported node: %r', node) - raise CompileError( - 'Unsupported ast node: {}'.format(ast.dump(node)), node) - - def visit_Attribute(self, node: ast.Attribute): - """Resolve refrences to attributes of objects (such as a.b.c)""" - return '{}.{}'.format(self.visit(node.value), node.attr) - - def visit_If(self, node: ast.If): - """Return the C representation of a python if statement""" - # Compile the test part of the if block into C code. This is the part - # coming directly after "if", which tests some boolean case. - test_src = self.visit(node.test) - - # Compile the body to C source. This is the part executed when the test - # evaluates to True. - body_src = '' - for body_node in node.body: - body_src += self.visit(body_node) - - # Compile the orelse to C source. This is the part executed when the - # test evaluates to False. - if node.orelse: - orelse_src = '' - for orelse_node in node.orelse: - orelse_src += self.visit(orelse_node) - - # Return all compiled source in an if...else... format - return 'if ({test}) {{\n{body}\n}} else {{\n{orelse}\n}}'.format( - test=test_src, body=body_src, orelse=orelse_src) - else: - # If there wasn't an else block, return a simpler format - return 'if ({test}) {{\n{body}\n}}'.format( - test=test_src, body=body_src) + raise CompileError('unhandled visit: {!r}'.format(node)) - def visit_Eq(self, node: ast.Eq): - return '==' + def compiler(self) -> str: + raise NotImplementedError() - def visit_Gt(self, node: ast.Gt): - return '>' - def visit_Lt(self, node: ast.Lt): - return '<' +class LineCompiler(BaseCompiler): + def visit_Return(self, ret_node): + return 'return {};'.format(self.visit(ret_node.value)) - def visit_BoolOp(self, node: ast.BoolOp): - return ' {} '.format(self.visit(node.op)).join([ - '({})'.format(self.visit(v)) for v in node.values]) + def visit_Num(self, num_node): + return str(num_node.n) - def visit_BinOp(self, node: ast.BinOp): - left_src = self.visit(node.left) - right_src = self.visit(node.right) - op_src = self.visit(node.op) - return '{left} {op} {right}'.format( - left=left_src, op=op_src, right=right_src) + def compile(self) -> str: + c_src = self.visit(self.root) + return c_src - def visit_Add(self, node: ast.Add): - return '+' - def visit_And(self, node: ast.And): - return '&&' +class FuncCompiler(BaseCompiler): + def compile(self) -> str: + c_src = 'int {}() {{\n'.format(self.name) + for node in self.root.body: + line_comp = LineCompiler('{}:{}'.format(node.lineno, node.col_offset), node) + line_c_src = line_comp.compile() + c_src += ' ' + line_c_src + '\n' + LOG.debug('line_c_src: %s', line_c_src) + c_src += '}\n' + return c_src - def visit_Or(self, node: ast.Or): - return '||' - def visit_Compare(self, node:ast.Compare): - """Return the C representation of comparison tests""" - # Find the C type of the left value - _, left_val = self._load_name(node.left) - left_type = self._ctype(left_val) +class ModuleCompiler(BaseCompiler): + def compile(self) -> str: + c_src = '' + func_nodes = [] + other_nodes = [] - # Find the C type of the right values and ensure they are same as left - for cmp in node.comparators: - # Lookup ast.Name nodes first, then convert their value to a type - if type(cmp) == ast.Name: - _, right_val = self._load_name(cmp) - right_type = self._ctype(right_val) + # Sort the body nodes by type (top-level code or functions) + for node in self.root.body: + if type(node) == ast.FunctionDef: + func_nodes.append(node) else: - # For other things like ast.Str and ast.Num just convert to a - # type directly - right_type = self._ctype(cmp) - - # Enforce that the two are the same C type - if left_type != right_type: - raise CompileError('mismatched types in comparison', node.left) - - # String comparison is a different matter in C - if left_type == BUILTIN_TYPES[PYTYPE_STR]: - # Ensure exactly two items are being compared (left + 1 comparator) - if len(node.comparators) != 1: - raise CompileError( - 'string comparisons must be of exactly two items', node) - - # Determine which strcmp() return value to expect - op = node.ops[0] - comp = '==' - if type(op) == ast.Eq: - expect = 0 - elif type(op) == ast.NotEq: - expect = 0 - comp = '!=' - elif type(op) == ast.Lt: - expect = -1 - elif type(op) == ast.Gt: - expect = 1 - else: - raise CompileError( - 'unsupported string comparison op: {}' - .format(ast.dump(op)), node) - - return 'strcmp({}, {}) {} {}'.format( - self.visit(node.left), self.visit(node.comparators[0]), comp, - expect) - else: - # For non-string comparisons just translate the symbols - parts = [] - parts.append(self.visit(node.left)) - for op, cmp in zip(node.ops, node.comparators): - parts.append(self.visit(op)) - parts.append(self.visit(cmp)) - LOG.debug('compare parts: %r', parts) - return ' '.join(parts) + other_nodes.append(node) - def visit_Call(self, node:ast.Call): - # TODO: Check that the function exists - # TODO: Check arguments - cargs = [] - for arg in node.args: - cargs.append(self.visit(arg)) - return '{}({})'.format(node.func.id, ', '.join(cargs)) + # Compile the top-level module code + init_func_def = ast.FunctionDef( + name='{}DOT__init__'.format(self.name), + body=other_nodes) + init_func_compiler = FuncCompiler(init_func_def.name, init_func_def) + c_src += init_func_compiler.compile() - def visit_Str(self, node:ast.Str): - """Return the C representation of a python string""" - # Convert strings to hex byte arrays, and include a null termination - return '(const char[]){{{}}}'.format( - ', '.join([hex(ord(c)) for c in node.s + '\0'])) + return c_src + - def visit_Name(self, node:ast.Name): - """Returns the C name of a python variable""" - # _load_name returns both the cname and the ast node, but we only need - # the name. - return node.id +class ProgramCompiler(object): + def __init__(self, py_src): + self.py_src = py_src - def visit_NameConstant(self, node: ast.NameConstant): - """Returns the C name of a python constant""" - # Check if the node refers to the python constant 'True' - if node.value == True: - return 'true' - # Check if the node refers to the python constant 'False' - elif node.value == False: - return 'false' - # There are certainly other constants with no implementation here, so - # just raise an error - else: - raise CompileError( - 'could not compile constant `{}`'.format(ast.dump(node)), node) + def compile(self) -> str: + root = ast.parse(self.py_src) + main_comp = ModuleCompiler('__main__', root) + c_src = '' - def visit_Num(self, node: ast.Num): - """Return the C representation of a python numerical value""" - # Numbers are represented just the same in C as they are in python, so - # just convert to string and return the C representation - return str(node.n) - - def visit_Return(self, node: ast.Return): - """Return the C representation of a python 'return' statement""" - # Ensure the value being returned matches the annotated type of this - # function. - # TODO: Check that the type is valid to return - return 'return {};'.format(self.visit(node.value)) - - def visit_Expr(self, node: ast.Expr): - """Return the C representation of a python expression""" - return self.visit(node.value) + ';' - - def visit_Import(self, node: ast.Import): - LOG.debug(ast.dump(node)) - for alias in node.names: - return '#include "{}.h"\n'.format(alias.name) - - def visit_Pass(self, node: ast.Pass): - # Unlike python, no special keywords are required for a NOP body, so we - # don't actually need to do anything here - pass - - def visit_FunctionDef(self, node: ast.FunctionDef): - if type(self.node) != ast.Module: - raise CompileError('Inner functions are not supported', node) - - def visit_Assign(self, node: ast.Assign): - # Python allows multiple assignments on one line, but that - # isn't implemented here yet. - if len(node.targets) > 1: - raise CompileError( - 'Use of unsupported feature: multiple assignment', node.targets) - target = node.targets[0] - - # Do not support attribute assignment (object.x = 123) - if type(target) == ast.Attribute: - raise CompileError( - 'Use of unsupported feature: attribute assignment', node) - - # Ensure the variable has been declared - if not self.scope.contains(target.id): - raise CompileError( - 'Cannot assign to undeclared local var `{}`' - .format(target.id), node) - - # Handle assingent of numerical constants to variables - if type(node.value) == ast.Num: - # ensure target is an int - if self.scope.get(target.id).pytype() != PYTYPE_INT: - raise CompileError( - 'assignment of int to incompatible {} var {}' - .format(self.scope.get(target.id), target.id), node.value) - - # prevent float assignment - if '.' in str(node.value.n): - raise CompileError( - 'assignment of float to incompatible {} var {}' - .format(self.scope.get(target.id), target.id), node.value) - - # output the code - return '{} = {};\n'.format(target.id, node.value.n) - - # Handle assignment of string constants to variables - elif type(node.value) == ast.Str: - # ensure target is a string - if self.scope[target.id] != PYTYPE_STR: - raise CompileError( - 'assignment of str to incompatible {} var `{}`' - .format(self.scope.get(target.id), target.id), node.value) - - # output the code - return '{} = "{}";\n'.format( - target.id, node.value.s.replace('"', '\\"')) - else: - raise CompileError( - 'Unsupported assignment of type `{}` to `{}` of type `{}`' - .format(type(node.value), target.id, self.scope.get(target.id)), - node.value) - - def visit_AnnAssign(self, node: ast.AnnAssign): - # Sort out whether this is a new local declaration - if self.scope.contains(node.target.id): - raise CompilerError( - 'Local var `{}` has already been declared' - .format(node.target.id)) - - # TODO: Ensure the data types match for assignment - - # Store this declaration in the locals table - # TODO: Do not assume node.annotation has an `id` attr - self.scope.set(node.target.id, pytype_from_str(node.annotation.id)) - - # Generate C code for the assignment - target_src = self.visit(node.target) - value_src = self.visit(node.value) - return 'int {} = {};'.format(target_src, value_src) - - -class ModuleCompiler(ast.NodeVisitor): - def __init__(self, module_name, source_filename, node, dunder_name): - self.module_name = module_name - self.scope = Scope() - self.node = node - self.source_filename = source_filename - self.__name__ = dunder_name - - # Declare __name__ as a string global - self.scope.set('__name__', StrType) - - def _initial_module_source(self): - return '\n'.join([ - '#include ', - '#define true 1', - '#define false 0', - 'extern int strcmp(const char*, const char*);', - 'extern int printf(const char*, ...);', - '#define {prefix}{mod_name}{DOT}__name__ "{dunder_name}"'.format( - prefix=MOD_PREFIX, mod_name=self.module_name, DOT=DOT, - dunder_name=self.__name__), - ]) + '\n\n' - - def generic_visit(self, node): - raise CompileError( - 'No matching compiler handler for node {!r}' - .format(node), node) - - def compile(self): - # Store the generated C source code for program in one big string - src = '' - - # Look for import statements - for node in ast.walk(self.node): - if type(node) != ast.Import: - continue - - # Look at what module is being imported - for alias in node.names: - # If the import isn't the form of "import ... as ..." then - # juse use the normal name for the missing "as ..." - asname = alias.asname if alias.asname else alias.name - - # Ensure the asname isn't taken - if self.scope.contains(asname): - raise CompileError( - 'cannot import `{}` multiple times'.format( - asname), node) - - # Look for the module relative to the current working dir - filename = alias.name.replace('.', '/') + '.py' - - # Open the file - with open(filename) as fh: - # Parse the root node using the ast module - root_node = ast.parse(fh.read()) - - # Create a compiler for this module - compiler = ModuleCompiler( - alias.name, filename, root_node, asname) - - # Ensure the module is initialized later - imported_modules.append(asname) - - # Expose that module as a global in this module - self.scope.set(asname, ModuleType(scope=compiler.scope)) - - # Compile the source and add it to the current source string - src += compiler.compile() - - # Add some pre-code for the module - src += self._initial_module_source() - - # Keep track of compilers for the functions in this module - func_compilers = [] - - # Build a compiler for the top-level function - top_func_compiler = FunctionCompiler(self.module_name, self, self.node) - func_compilers.append(top_func_compiler) - - # Build a compiler for all other functions - for mod_node in self.node.body: - if type(mod_node) == ast.FunctionDef: - self.scope.set(mod_node.name, FuncType(args=mod_name.args)) - func_compiler = FunctionCompiler( - self.module_name, self, mod_node) - func_compilers.append(func_compiler) - - # Run the compilers - try: - for compiler in func_compilers: - src += compiler.compile() + '\n' - except CompileError as e: - e._msg = '{}:{}'.format(self.source_filename, e._msg) - raise - - # Return the generated C source code - return src + c_src += main_comp.compile() + c_src += 'int main() {return __main__DOT__init__();}' + return c_src def parse_args(): p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - 'input_modules', nargs='+', help='Python module names for compilation') + p.add_argument('sourcefile', type=argparse.FileType('r')) return p.parse_args() def main(): - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig() args = parse_args() - src = '' - for i, module_name in enumerate(args.input_modules): - # Figure out what __name__ will be for this module. The first module - # listed will be __main__. - if i == 0: - dunder_name = '__main__' - else: - dunder_name = module_name - - # Build a path name for the python module - filename = module_name.replace('.', '/') + '.py' - - # Open parse, and compile the file - with open(filename) as fh: - # Parse the file using standard python parser which gives back a - # data structure called an AST representing the code - module = ast.parse(fh.read()) - - # Using the returned AST generate C code - compiler = ModuleCompiler( - module_name, filename, module, dunder_name) - - # Add the generated C code to the project source - src += compiler.compile() - - # Add a main() fn, calling the first module given on the CLI - # TODO: initialize imported functions in main() or somewhere similar - src += 'int main() {{return {}{}{}();}}\n'.format( - MOD_PREFIX, args.input_modules[0], MOD_INIT_SUFFIX) - - print(src) + py_src = args.sourcefile.read() + prog_compiler = ProgramCompiler(py_src) + print(prog_compiler.compile()) return os.EX_OK From b6eefb200c51ef73c880edbf16cb3b31d88d53f4 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:16:04 +0100 Subject: [PATCH 40/66] Simplify test suite Do not use exit code 0 as indication of good test as its too easily found by accident. --- main.py | 12 ------------ run_tests.sh | 6 +++--- tests/and/test.py | 11 ----------- tests/int/run.sh | 4 ---- tests/int/test.py | 3 --- tests/{and => return}/run.sh | 3 ++- tests/return/test.py | 1 + 7 files changed, 6 insertions(+), 34 deletions(-) delete mode 100644 main.py delete mode 100644 tests/and/test.py delete mode 100644 tests/int/run.sh delete mode 100644 tests/int/test.py rename tests/{and => return}/run.sh (59%) create mode 100644 tests/return/test.py diff --git a/main.py b/main.py deleted file mode 100644 index 6c4e0123..00000000 --- a/main.py +++ /dev/null @@ -1,12 +0,0 @@ -def main() -> int: - z: int = get42() - x: int = 30 - if z > x: - print('Hello World') - return x - -def get42() -> int: - return 42 - -if __name__ == "__main__": - return main() diff --git a/run_tests.sh b/run_tests.sh index 71d3cca6..d0935f34 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,12 +1,12 @@ #!/bin/bash -export GCC=gcc -export PYC=$(pwd)/pyc.py +export GCC="gcc -Wall -pedantic-errors" +export PYC="$(pwd)/pyc.py" FAILED=0 PASSED=0 for test_script in $(find $(pwd)/tests -name run.sh); do (cd $(dirname $test_script); bash -e $test_script) - if [ $? -ne 0 ]; then + if [ $? -ne 42 ]; then echo "FAILED: $test_script" let FAILED++ else diff --git a/tests/and/test.py b/tests/and/test.py deleted file mode 100644 index e76b402e..00000000 --- a/tests/and/test.py +++ /dev/null @@ -1,11 +0,0 @@ -if True and True: - print("OK\n") - -if False and True: - print("ERR\n") - -if False and False: - print("ERR\n") - -if True and False: - print("ERR\n") diff --git a/tests/int/run.sh b/tests/int/run.sh deleted file mode 100644 index c1ffc148..00000000 --- a/tests/int/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -${PYC} test > test.c -${GCC} test.c -otest.bin -./test.bin diff --git a/tests/int/test.py b/tests/int/test.py deleted file mode 100644 index a04f655e..00000000 --- a/tests/int/test.py +++ /dev/null @@ -1,3 +0,0 @@ -x: int = 1 -y: int = 34 -print(x + y) diff --git a/tests/and/run.sh b/tests/return/run.sh similarity index 59% rename from tests/and/run.sh rename to tests/return/run.sh index c1ffc148..a5a2f11b 100644 --- a/tests/and/run.sh +++ b/tests/return/run.sh @@ -1,4 +1,5 @@ #!/bin/sh -${PYC} test > test.c +${PYC} test.py > test.c ${GCC} test.c -otest.bin ./test.bin +exit $? \ No newline at end of file diff --git a/tests/return/test.py b/tests/return/test.py new file mode 100644 index 00000000..e71fa1ad --- /dev/null +++ b/tests/return/test.py @@ -0,0 +1 @@ +return 42 \ No newline at end of file From ef861c2f3924bd6c93ca98baf198f9ee5d131174 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:21:03 +0100 Subject: [PATCH 41/66] change error formating --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 15b36921..1ed1240f 100755 --- a/pyc.py +++ b/pyc.py @@ -21,7 +21,7 @@ def __init__(self, name, root): pass def generic_visit(self, node): - raise CompileError('unhandled visit: {!r}'.format(node)) + raise CompileError('unhandled visit: {}'.format(ast.dump(node))) def compiler(self) -> str: raise NotImplementedError() From 5fbeb81c24f0c90a156db808a7c7524489610666 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:21:08 +0100 Subject: [PATCH 42/66] new line at end --- pyc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyc.py b/pyc.py index 1ed1240f..f910a46d 100755 --- a/pyc.py +++ b/pyc.py @@ -105,3 +105,4 @@ def main(): if __name__ == '__main__': sys.exit(main()) + \ No newline at end of file From a081a6bef7c7cf17fbb3a21c50eaba003dfdf0f9 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:22:42 +0100 Subject: [PATCH 43/66] Add new line at end of file --- tests/return/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/return/test.py b/tests/return/test.py index e71fa1ad..4c93c1e7 100644 --- a/tests/return/test.py +++ b/tests/return/test.py @@ -1 +1 @@ -return 42 \ No newline at end of file +return 42 From 9cd832762da8156b57c7a33707cbc442b119d21b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sat, 9 Dec 2017 03:23:09 +0100 Subject: [PATCH 44/66] Add new line at end of file --- tests/return/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/return/run.sh b/tests/return/run.sh index a5a2f11b..e597f23d 100644 --- a/tests/return/run.sh +++ b/tests/return/run.sh @@ -2,4 +2,4 @@ ${PYC} test.py > test.c ${GCC} test.c -otest.bin ./test.bin -exit $? \ No newline at end of file +exit $? From c821507afe4bccf55fc1e169dd386e1e38dcd85f Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:07:28 +0100 Subject: [PATCH 45/66] Add test which returns a constant int --- tests/{return => return.constant.int}/run.sh | 0 tests/{return => return.constant.int}/test.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{return => return.constant.int}/run.sh (100%) rename tests/{return => return.constant.int}/test.py (100%) diff --git a/tests/return/run.sh b/tests/return.constant.int/run.sh similarity index 100% rename from tests/return/run.sh rename to tests/return.constant.int/run.sh diff --git a/tests/return/test.py b/tests/return.constant.int/test.py similarity index 100% rename from tests/return/test.py rename to tests/return.constant.int/test.py From 1e02c3a8ff690a76c845cdacce57cd46d1c2df9a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:07:39 +0100 Subject: [PATCH 46/66] Add test which returns variable int --- tests/return.var.int/run.sh | 5 +++++ tests/return.var.int/test.py | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 tests/return.var.int/run.sh create mode 100644 tests/return.var.int/test.py diff --git a/tests/return.var.int/run.sh b/tests/return.var.int/run.sh new file mode 100644 index 00000000..e597f23d --- /dev/null +++ b/tests/return.var.int/run.sh @@ -0,0 +1,5 @@ +#!/bin/sh +${PYC} test.py > test.c +${GCC} test.c -otest.bin +./test.bin +exit $? diff --git a/tests/return.var.int/test.py b/tests/return.var.int/test.py new file mode 100644 index 00000000..3620a6f6 --- /dev/null +++ b/tests/return.var.int/test.py @@ -0,0 +1,2 @@ +x: int = 42 +return x From fc4be2c3a1f30d2694fb9213aa59f2ece0c284a6 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:08:18 +0100 Subject: [PATCH 47/66] Enable debug logging --- pyc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index f910a46d..be8e21e1 100755 --- a/pyc.py +++ b/pyc.py @@ -94,8 +94,8 @@ def parse_args(): return p.parse_args() -def main(): - logging.basicConfig() +def main() -> int: + logging.basicConfig(level=logging.DEBUG) args = parse_args() py_src = args.sourcefile.read() prog_compiler = ProgramCompiler(py_src) From 17e96643e8e1ab17604c6c0a54528be4d87fe370 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:08:30 +0100 Subject: [PATCH 48/66] Add type annotation to parse_args() --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index be8e21e1..19feb7b5 100755 --- a/pyc.py +++ b/pyc.py @@ -88,7 +88,7 @@ def compile(self) -> str: return c_src -def parse_args(): +def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) p.add_argument('sourcefile', type=argparse.FileType('r')) return p.parse_args() From e2c3941f9c0ad35198ee999c6348c1ddd8291774 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:12:06 +0100 Subject: [PATCH 49/66] Add \n at end of file --- pyc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyc.py b/pyc.py index 19feb7b5..b51a9dca 100755 --- a/pyc.py +++ b/pyc.py @@ -105,4 +105,3 @@ def main() -> int: if __name__ == '__main__': sys.exit(main()) - \ No newline at end of file From 7b8471b983990c2aded733a79d4f9bbd1dc3a17a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 01:12:51 +0100 Subject: [PATCH 50/66] Add int and scope handling This makes return.var.int test pass. --- pyc.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-) diff --git a/pyc.py b/pyc.py index b51a9dca..fed9653f 100755 --- a/pyc.py +++ b/pyc.py @@ -5,14 +5,41 @@ import os import sys +from collections import namedtuple + LOG = logging.getLogger(__name__) +ScopeEntry = namedtuple('ScopeEntry', ['c_name', 'c_type', 'py_name', 'py_type']) + +class Scope(dict): + def __init__(self, parent=None, prefix=None): + self.prefix = prefix + if parent: + dict.__init__(self, parent) + else: + dict.__init__(self) + + def add_entry(self, py_name, py_type, c_name, c_type): + self[py_name] = ScopeEntry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) + + def c_name(self, name): + if self.prefix: + return '{}_DOT_{}'.format(self.prefix, name) + else: + return name + + def dict(self) -> dict: + return dict(self) + +BUILTIN = Scope() + class CompileError(RuntimeError): pass class BaseCompiler(ast.NodeVisitor): - def __init__(self, name, root): + def __init__(self, name, root, scope): self.name = name self.root = root + self.scope = scope self.docstring = '' try: if type(root.body[0]) == ast.Str: @@ -26,14 +53,49 @@ def generic_visit(self, node): def compiler(self) -> str: raise NotImplementedError() + def declare_var(self, node: ast.AnnAssign) -> str: + py_name = node.target.id + py_type = node.annotation.id + c_name = self.scope.c_name(py_name) + if py_type == 'int': + c_type = 'int32_t' + else: + raise NotImplementedError('unhandled py_type: {}'.format(py_type)) + self.scope.add_entry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) + LOG.debug('set scope entry `%s` in compiler %s', py_name, self.name) + return '{c_type} {c_name};'.format( + c_type=c_type, c_name=c_name) + class LineCompiler(BaseCompiler): - def visit_Return(self, ret_node): + def visit_Return(self, ret_node) -> str: return 'return {};'.format(self.visit(ret_node.value)) - def visit_Num(self, num_node): + def visit_Num(self, num_node) -> str: return str(num_node.n) + def visit_AnnAssign(self, node): + py_name = node.target.id + if py_name not in self.scope: + raise CompileError('assignment to undeclared variable `{}` in scope {!r}'.format(py_name, self.scope)) + decl = self.scope[py_name] + if decl.py_type == 'int': + if type(node.value) != ast.Num: + raise CompileError( + 'assignment of non-numerical value {} to int variable `{}`' + .format(ast.dump(node), py_name)) + value = int(node.value.n) + else: + raise NotImplementedError('unhandled py_type: {}'.format(decl.py_type)) + return '{c_name} = {value};'.format( + c_name=decl.c_name, value=value) + + def visit_Name(self, node): + py_name = node.id + if py_name not in self.scope: + raise CompileError('unknown reference `{}`'.format(py_name)) + return self.scope[py_name].c_name + def compile(self) -> str: c_src = self.visit(self.root) return c_src @@ -43,15 +105,16 @@ class FuncCompiler(BaseCompiler): def compile(self) -> str: c_src = 'int {}() {{\n'.format(self.name) for node in self.root.body: - line_comp = LineCompiler('{}:{}'.format(node.lineno, node.col_offset), node) + line_name = '{}:{}'.format(node.lineno, node.col_offset) + line_comp = LineCompiler(line_name, node, self.scope) line_c_src = line_comp.compile() c_src += ' ' + line_c_src + '\n' LOG.debug('line_c_src: %s', line_c_src) - c_src += '}\n' + c_src += '}\n\n' return c_src -class ModuleCompiler(BaseCompiler): +class ModuleCompiler(BaseCompiler): def compile(self) -> str: c_src = '' func_nodes = [] @@ -64,25 +127,47 @@ def compile(self) -> str: else: other_nodes.append(node) + # Find all module level variable declarations + for node in other_nodes: + if type(node) != ast.AnnAssign: + continue + c_src += self.declare_var(node) + # Compile the top-level module code init_func_def = ast.FunctionDef( name='{}DOT__init__'.format(self.name), + annotation=ast.Name(id='int'), body=other_nodes) - init_func_compiler = FuncCompiler(init_func_def.name, init_func_def) + init_func_compiler = FuncCompiler(init_func_def.name, init_func_def, self.scope) c_src += init_func_compiler.compile() return c_src class ProgramCompiler(object): - def __init__(self, py_src): + def __init__(self, name, py_src): + self.name = name self.py_src = py_src + def _pre_source(self) -> str: + return '\n'.join(['#include ']) + '\n\n' + def compile(self) -> str: + # Use CPython's builtin source parser root = ast.parse(self.py_src) - main_comp = ModuleCompiler('__main__', root) - c_src = '' + # The module exists in a new scope which inherits all builtin declarations + main_scope = Scope(BUILTIN) + + # In the __main__ module, __main__ is self-referential + main_scope['__main__'] = main_scope + + # Create a new compiler for the __main__ module + main_comp = ModuleCompiler('__main__', root, main_scope) + + # Create and return C source for the application, which can be compiled + # to binary form using gcc. + c_src = self._pre_source() c_src += main_comp.compile() c_src += 'int main() {return __main__DOT__init__();}' return c_src @@ -98,7 +183,8 @@ def main() -> int: logging.basicConfig(level=logging.DEBUG) args = parse_args() py_src = args.sourcefile.read() - prog_compiler = ProgramCompiler(py_src) + module_name = os.path.basename(args.sourcefile.name) + prog_compiler = ProgramCompiler(module_name, py_src) print(prog_compiler.compile()) return os.EX_OK From a6e5447aeff6037cc8dd5a8ee1f5ec493b76ed9b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:20:18 +0100 Subject: [PATCH 51/66] Set initial variable for C int's --- pyc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index fed9653f..58ea87ea 100755 --- a/pyc.py +++ b/pyc.py @@ -59,12 +59,13 @@ def declare_var(self, node: ast.AnnAssign) -> str: c_name = self.scope.c_name(py_name) if py_type == 'int': c_type = 'int32_t' + def_value = '0' else: raise NotImplementedError('unhandled py_type: {}'.format(py_type)) self.scope.add_entry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) LOG.debug('set scope entry `%s` in compiler %s', py_name, self.name) - return '{c_type} {c_name};'.format( - c_type=c_type, c_name=c_name) + return '{c_type} {c_name} = {def_value};'.format( + c_type=c_type, c_name=c_name, def_value=def_value) class LineCompiler(BaseCompiler): From 10821b285baf0f32db707a554010ebb0d46089b6 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:20:54 +0100 Subject: [PATCH 52/66] Fix whitespace on var declarations --- pyc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 58ea87ea..9615e9e6 100755 --- a/pyc.py +++ b/pyc.py @@ -132,7 +132,8 @@ def compile(self) -> str: for node in other_nodes: if type(node) != ast.AnnAssign: continue - c_src += self.declare_var(node) + c_src += self.declare_var(node) + '\n' + c_src += '\n' # Compile the top-level module code init_func_def = ast.FunctionDef( From 562a00b3317784eae917065eb7fb58325ecd5fff Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:21:04 +0100 Subject: [PATCH 53/66] #include --- pyc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyc.py b/pyc.py index 9615e9e6..a090490d 100755 --- a/pyc.py +++ b/pyc.py @@ -152,7 +152,7 @@ def __init__(self, name, py_src): self.py_src = py_src def _pre_source(self) -> str: - return '\n'.join(['#include ']) + '\n\n' + return '\n'.join(['#include ', '#include ']) + '\n\n' def compile(self) -> str: # Use CPython's builtin source parser From 771d5d61ce1922bb1494f0fc8e7fcf206f2f79d0 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:25:41 +0100 Subject: [PATCH 54/66] Annotate node param types --- pyc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyc.py b/pyc.py index a090490d..50adf77d 100755 --- a/pyc.py +++ b/pyc.py @@ -69,13 +69,13 @@ def declare_var(self, node: ast.AnnAssign) -> str: class LineCompiler(BaseCompiler): - def visit_Return(self, ret_node) -> str: + def visit_Return(self, ret_node: ast.Return) -> str: return 'return {};'.format(self.visit(ret_node.value)) - def visit_Num(self, num_node) -> str: + def visit_Num(self, num_node: ast.Num) -> str: return str(num_node.n) - def visit_AnnAssign(self, node): + def visit_AnnAssign(self, node: ast.AnnAssign): py_name = node.target.id if py_name not in self.scope: raise CompileError('assignment to undeclared variable `{}` in scope {!r}'.format(py_name, self.scope)) @@ -91,7 +91,7 @@ def visit_AnnAssign(self, node): return '{c_name} = {value};'.format( c_name=decl.c_name, value=value) - def visit_Name(self, node): + def visit_Name(self, node: ast.Name): py_name = node.id if py_name not in self.scope: raise CompileError('unknown reference `{}`'.format(py_name)) From 0d3f56366f9d23c9429670ce9ace54e034f475c0 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:30:06 +0100 Subject: [PATCH 55/66] Annotate return types on some funcs --- pyc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index 50adf77d..afbf31a6 100755 --- a/pyc.py +++ b/pyc.py @@ -75,7 +75,7 @@ def visit_Return(self, ret_node: ast.Return) -> str: def visit_Num(self, num_node: ast.Num) -> str: return str(num_node.n) - def visit_AnnAssign(self, node: ast.AnnAssign): + def visit_AnnAssign(self, node: ast.AnnAssign) -> str: py_name = node.target.id if py_name not in self.scope: raise CompileError('assignment to undeclared variable `{}` in scope {!r}'.format(py_name, self.scope)) @@ -91,7 +91,7 @@ def visit_AnnAssign(self, node: ast.AnnAssign): return '{c_name} = {value};'.format( c_name=decl.c_name, value=value) - def visit_Name(self, node: ast.Name): + def visit_Name(self, node: ast.Name) -> str: py_name = node.id if py_name not in self.scope: raise CompileError('unknown reference `{}`'.format(py_name)) From 62c7347baa43b59363af84e5974d501ddeacd161 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 02:46:39 +0100 Subject: [PATCH 56/66] Ensure test.py files are valid in python3.6 --- run_tests.sh | 1 + tests/return.constant.int/run.sh | 1 + tests/return.var.int/run.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index d0935f34..77966627 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,5 @@ #!/bin/bash +export PYTHON=python3.6 export GCC="gcc -Wall -pedantic-errors" export PYC="$(pwd)/pyc.py" diff --git a/tests/return.constant.int/run.sh b/tests/return.constant.int/run.sh index e597f23d..f8029661 100644 --- a/tests/return.constant.int/run.sh +++ b/tests/return.constant.int/run.sh @@ -1,4 +1,5 @@ #!/bin/sh +${PYTHON} test.py ${PYC} test.py > test.c ${GCC} test.c -otest.bin ./test.bin diff --git a/tests/return.var.int/run.sh b/tests/return.var.int/run.sh index e597f23d..f8029661 100644 --- a/tests/return.var.int/run.sh +++ b/tests/return.var.int/run.sh @@ -1,4 +1,5 @@ #!/bin/sh +${PYTHON} test.py ${PYC} test.py > test.c ${GCC} test.c -otest.bin ./test.bin From 64f7c35f904acdc805b1488fe103370448f8270a Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 13:07:15 +0100 Subject: [PATCH 57/66] Add string comparison and C calls --- pyc.py | 186 ++++++++++++++++++++++++++---- run_tests.sh | 10 +- tests/if.comp.str/run.sh | 13 +++ tests/if.comp.str/test.py | 12 ++ tests/return.constant.int/run.sh | 9 +- tests/return.constant.int/test.py | 5 +- tests/return.var.int/run.sh | 9 +- tests/return.var.int/test.py | 6 +- 8 files changed, 220 insertions(+), 30 deletions(-) create mode 100644 tests/if.comp.str/run.sh create mode 100644 tests/if.comp.str/test.py diff --git a/pyc.py b/pyc.py index afbf31a6..72fbe3ed 100755 --- a/pyc.py +++ b/pyc.py @@ -28,6 +28,24 @@ def c_name(self, name): else: return name + def resolve(self, node): + py_name_parts = [] + def add_node(sub_node): + if type(sub_node) == ast.Attribute: + add_node(sub_node.value) + py_name_parts.append(sub_node.attr) + elif type(sub_node) == ast.Name: + py_name_parts.append(sub_node.id) + else: + raise NotImplementedError('cannot resolve from {}'.format(ast.dump(sub_node))) + + add_node(node) + LOG.debug('py_name_parts: %r', py_name_parts) + cur = self + for py_name in py_name_parts: + cur = cur[py_name] + return cur + def dict(self) -> dict: return dict(self) @@ -60,6 +78,9 @@ def declare_var(self, node: ast.AnnAssign) -> str: if py_type == 'int': c_type = 'int32_t' def_value = '0' + elif py_type == 'str': + c_type = 'char*' + def_value = 'NULL' else: raise NotImplementedError('unhandled py_type: {}'.format(py_type)) self.scope.add_entry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) @@ -67,10 +88,17 @@ def declare_var(self, node: ast.AnnAssign) -> str: return '{c_type} {c_name} = {def_value};'.format( c_type=c_type, c_name=c_name, def_value=def_value) + def py_type(self, node): + if type(node) in [ast.Name, ast.Attribute]: + var = self.scope.resolve(node) + return var.py_type + else: + raise NotImplementedError('cannot get type of {}'.format(ast.dump(node))) + class LineCompiler(BaseCompiler): def visit_Return(self, ret_node: ast.Return) -> str: - return 'return {};'.format(self.visit(ret_node.value)) + return 'return {}'.format(self.visit(ret_node.value)) def visit_Num(self, num_node: ast.Num) -> str: return str(num_node.n) @@ -79,49 +107,158 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> str: py_name = node.target.id if py_name not in self.scope: raise CompileError('assignment to undeclared variable `{}` in scope {!r}'.format(py_name, self.scope)) + decl = self.scope[py_name] if decl.py_type == 'int': if type(node.value) != ast.Num: raise CompileError( 'assignment of non-numerical value {} to int variable `{}`' .format(ast.dump(node), py_name)) - value = int(node.value.n) + value_src = self.visit(node.value) + elif decl.py_type == 'str': + if type(node.value) != ast.Str: + raise CompileError( + 'assignment of non-string value {} to str variable `{}`' + .format(ast.dump(node), py_name)) + value_src = self.visit(node.value) else: raise NotImplementedError('unhandled py_type: {}'.format(decl.py_type)) - return '{c_name} = {value};'.format( - c_name=decl.c_name, value=value) + return '{c_name} = {value_src}'.format( + c_name=decl.c_name, value_src=value_src) def visit_Name(self, node: ast.Name) -> str: py_name = node.id + + # Allow any reference into the C namespace. Let C compiler check them. + if py_name == 'C': + return '' + if py_name not in self.scope: - raise CompileError('unknown reference `{}`'.format(py_name)) - return self.scope[py_name].c_name + raise CompileError('NameError: undefined reference `{}`'.format(py_name)) + + c_name = self.scope[py_name].c_name + LOG.debug('c_name(%r) == %s', py_name, c_name) + return c_name + + def visit_Attribute(self, node: ast.Attribute) -> str: + base_name = self.visit(node.value) + if base_name: + return '{}.{}'.format(self.visit(node.value), node.attr) + else: + return node.attr + + def visit_Import(self, node: ast.Attribute) -> str: + src = '' + for alias in node.names: + if alias.name == 'C': + continue + return src + + def visit_If(self, node: ast.Attribute) -> str: + # Compile the test condition + test_src = self.visit(node.test) + + # Compile the body + body_src = '' + for body_node in node.body: + line_name = '{}:{}'.format(body_node.lineno, body_node.col_offset) + line_comp = LineCompiler(line_name, body_node, self.scope) + body_src += line_comp.compile() + + # Compile the orelse body + orelse_src = '' + for orelse_node in node.orelse: + line_name = '{}:{}'.format(body_node.lineno, body_node.col_offset) + line_comp = LineCompiler(line_name, body_node, self.scope) + orelse_src += line_comp.compile() + + # Build the C version + if node.orelse: + return 'if ({test_src}) {{\n{body_src}\n}} else {{ {orelse_src} }}'.format( + test_src=test_src, body_src=body_src, orelse_src=orelse_src) + else: + return 'if ({test_src}) {{\n{body_src}\n}}'.format(test_src=test_src, body_src=body_src) + + def visit_Compare(self, node: ast.Compare) -> str: + left_py_type = self.py_type(node.left) + if left_py_type == 'str': + if len(node.ops) != 1: + raise CompileError('string comparison is only valid against a single comparator') + op = type(node.ops[0]) + comp = node.comparators[0] + + if op == ast.Eq: + c_test = '== 0' + elif op == ast.Lt: + c_test = '== -1' + elif op == ast.Gt: + c_test = '== 1' + elif op == ast.NotEq: + c_test = '!= 0' + else: + raise CompileError('invalid string comparison operator {}'.format(ast.dump(op))) + + left_src = self.visit(node.left) + right_src = self.visit(comp) + return 'strcmp({}, {}) {}'.format(left_src, right_src, c_test) + else: + parts = [self.visit(node.left)] + for op, comp in zip(node.ops, node.comparators): + parts.append(self.visit(op)) + parts.append(self.visit(comp)) + return ' '.join(parts) + + def visit_Eq(self, node: ast.Eq) -> str: + return '==' + + def visit_Str(self, node: ast.Str) -> str: + return '"{}"'.format(str(node.s)) + + def visit_Call(self, node: ast.Call) -> str: + arg_src_parts = [] + for arg in node.args: + arg_src_parts.append(self.visit(arg)) + args_src = ', '.join(arg_src_parts) + return '{}({})'.format(self.visit(node.func), args_src) + + def visit_Expr(self, node: ast.Expr) -> str: + src = self.visit(node.value) + if node == self.root: + return src + else: + return '(' + src + ')' def compile(self) -> str: c_src = self.visit(self.root) - return c_src + LOG.debug('compiled line:\n\t\t%s\n\n\tinto:\n\t\t%s', ast.dump(self.root), c_src) + if type(self.root) in [ast.Expr, ast.AnnAssign]: + c_src += ';' + c_src += '\n' + return c_src class FuncCompiler(BaseCompiler): def compile(self) -> str: - c_src = 'int {}() {{\n'.format(self.name) + c_src = 'void {}() {{\n'.format(self.name) for node in self.root.body: line_name = '{}:{}'.format(node.lineno, node.col_offset) line_comp = LineCompiler(line_name, node, self.scope) - line_c_src = line_comp.compile() - c_src += ' ' + line_c_src + '\n' - LOG.debug('line_c_src: %s', line_c_src) + c_src += line_comp.compile() c_src += '}\n\n' return c_src class ModuleCompiler(BaseCompiler): def compile(self) -> str: - c_src = '' - func_nodes = [] - other_nodes = [] + # Add a var for __name__ + dunder_name_c_name = self.scope.c_name('__name__') + self.scope.add_entry( + py_type='str', py_name='__name__', c_type='const char*', c_name=dunder_name_c_name) + c_src = 'const char* {} = "{}";\n'.format(dunder_name_c_name, self.name.replace('.', '_DOT_')) # Sort the body nodes by type (top-level code or functions) + func_nodes = [] + other_nodes = [] for node in self.root.body: if type(node) == ast.FunctionDef: func_nodes.append(node) @@ -130,9 +267,10 @@ def compile(self) -> str: # Find all module level variable declarations for node in other_nodes: - if type(node) != ast.AnnAssign: - continue - c_src += self.declare_var(node) + '\n' + for sub_node in ast.walk(node): + if type(sub_node) != ast.AnnAssign: + continue + c_src += self.declare_var(sub_node) + '\n' c_src += '\n' # Compile the top-level module code @@ -152,26 +290,30 @@ def __init__(self, name, py_src): self.py_src = py_src def _pre_source(self) -> str: - return '\n'.join(['#include ', '#include ']) + '\n\n' + return '\n'.join([ + '#include ', + '#include ', + '#include ',]) + '\n\n' def compile(self) -> str: # Use CPython's builtin source parser root = ast.parse(self.py_src) + module_name = '__main__' # The module exists in a new scope which inherits all builtin declarations - main_scope = Scope(BUILTIN) + main_scope = Scope(BUILTIN, prefix='MOD_{}'.format(module_name)) # In the __main__ module, __main__ is self-referential - main_scope['__main__'] = main_scope + main_scope[module_name] = main_scope # Create a new compiler for the __main__ module - main_comp = ModuleCompiler('__main__', root, main_scope) + main_comp = ModuleCompiler(module_name, root, main_scope) # Create and return C source for the application, which can be compiled # to binary form using gcc. c_src = self._pre_source() c_src += main_comp.compile() - c_src += 'int main() {return __main__DOT__init__();}' + c_src += 'int main() {__main__DOT__init__(); return 0;}' return c_src diff --git a/run_tests.sh b/run_tests.sh index 77966627..af26d9a0 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,10 +5,14 @@ export PYC="$(pwd)/pyc.py" FAILED=0 PASSED=0 -for test_script in $(find $(pwd)/tests -name run.sh); do - (cd $(dirname $test_script); bash -e $test_script) - if [ $? -ne 42 ]; then +for test_script in $(find tests -name run.sh); do + pushd $(dirname $test_script) + bash -xe ./run.sh + RETVAL=$? + popd + if [ $RETVAL -ne 42 ]; then echo "FAILED: $test_script" + echo "RETVAL: $RETVAL" let FAILED++ else echo "PASSED: $test_script" diff --git a/tests/if.comp.str/run.sh b/tests/if.comp.str/run.sh new file mode 100644 index 00000000..93a61409 --- /dev/null +++ b/tests/if.comp.str/run.sh @@ -0,0 +1,13 @@ +#!/bin/sh +${PYC} test.py > test.c +if [ $? -ne 0 ]; then + exit 1 +fi + +${GCC} test.c -otest.bin +if [ $? -ne 0 ]; then + exit 1 +fi + +./test.bin +exit $? diff --git a/tests/if.comp.str/test.py b/tests/if.comp.str/test.py new file mode 100644 index 00000000..c8fd9fcc --- /dev/null +++ b/tests/if.comp.str/test.py @@ -0,0 +1,12 @@ +import C +foo: str = 'foo' +bar: str = 'bar' + +if __name__ == '__main__': + if foo != 'foo': + C.exit(1) + + if foo == bar: + C.exit(2) + + C.exit(42) diff --git a/tests/return.constant.int/run.sh b/tests/return.constant.int/run.sh index f8029661..93a61409 100644 --- a/tests/return.constant.int/run.sh +++ b/tests/return.constant.int/run.sh @@ -1,6 +1,13 @@ #!/bin/sh -${PYTHON} test.py ${PYC} test.py > test.c +if [ $? -ne 0 ]; then + exit 1 +fi + ${GCC} test.c -otest.bin +if [ $? -ne 0 ]; then + exit 1 +fi + ./test.bin exit $? diff --git a/tests/return.constant.int/test.py b/tests/return.constant.int/test.py index 4c93c1e7..ea6cdf89 100644 --- a/tests/return.constant.int/test.py +++ b/tests/return.constant.int/test.py @@ -1 +1,4 @@ -return 42 +import C + +if __name__ == '__main__': + C.exit(42) diff --git a/tests/return.var.int/run.sh b/tests/return.var.int/run.sh index f8029661..93a61409 100644 --- a/tests/return.var.int/run.sh +++ b/tests/return.var.int/run.sh @@ -1,6 +1,13 @@ #!/bin/sh -${PYTHON} test.py ${PYC} test.py > test.c +if [ $? -ne 0 ]; then + exit 1 +fi + ${GCC} test.c -otest.bin +if [ $? -ne 0 ]; then + exit 1 +fi + ./test.bin exit $? diff --git a/tests/return.var.int/test.py b/tests/return.var.int/test.py index 3620a6f6..463300fc 100644 --- a/tests/return.var.int/test.py +++ b/tests/return.var.int/test.py @@ -1,2 +1,4 @@ -x: int = 42 -return x +import C +if __name__ == '__main__': + x: int = 42 + C.exit(x) From e8fe5e33c5f39471e0c9e1406450ef8c0e89192b Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Sun, 10 Dec 2017 21:00:23 +0100 Subject: [PATCH 58/66] Add comment --- pyc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyc.py b/pyc.py index 72fbe3ed..b07c55a5 100755 --- a/pyc.py +++ b/pyc.py @@ -150,6 +150,7 @@ def visit_Attribute(self, node: ast.Attribute) -> str: def visit_Import(self, node: ast.Attribute) -> str: src = '' for alias in node.names: + # The 'C' module has an internal implementation if alias.name == 'C': continue return src From 9377ea4d9d508ab28118829207d48277f2365729 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 12 Dec 2017 13:17:11 +0100 Subject: [PATCH 59/66] Use suggest_c_name() for main() --- pyc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pyc.py b/pyc.py index b07c55a5..8bd68536 100755 --- a/pyc.py +++ b/pyc.py @@ -23,10 +23,11 @@ def add_entry(self, py_name, py_type, c_name, c_type): self[py_name] = ScopeEntry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) def c_name(self, name): + def suggest_c_name(self, py_name): if self.prefix: - return '{}_DOT_{}'.format(self.prefix, name) + return '{}_DOT_{}'.format(self.prefix, py_name) else: - return name + return py_name def resolve(self, node): py_name_parts = [] @@ -240,7 +241,7 @@ def compile(self) -> str: class FuncCompiler(BaseCompiler): def compile(self) -> str: - c_src = 'void {}() {{\n'.format(self.name) + c_src = 'void {}() {{\n'.format(self.scope.suggest_c_name(self.name)) for node in self.root.body: line_name = '{}:{}'.format(node.lineno, node.col_offset) line_comp = LineCompiler(line_name, node, self.scope) @@ -276,7 +277,7 @@ def compile(self) -> str: # Compile the top-level module code init_func_def = ast.FunctionDef( - name='{}DOT__init__'.format(self.name), + name='__init__', annotation=ast.Name(id='int'), body=other_nodes) init_func_compiler = FuncCompiler(init_func_def.name, init_func_def, self.scope) @@ -314,7 +315,8 @@ def compile(self) -> str: # to binary form using gcc. c_src = self._pre_source() c_src += main_comp.compile() - c_src += 'int main() {__main__DOT__init__(); return 0;}' + c_src += 'int main() {{{}(); return 0;}}'.format( + main_scope.suggest_c_name('__init__')) return c_src From 9e724d14dae0d540c747a55ed9168de50808a764 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 12 Dec 2017 13:17:24 +0100 Subject: [PATCH 60/66] Fix SyntaxError --- pyc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyc.py b/pyc.py index 8bd68536..f4f4c261 100755 --- a/pyc.py +++ b/pyc.py @@ -22,7 +22,6 @@ def __init__(self, parent=None, prefix=None): def add_entry(self, py_name, py_type, c_name, c_type): self[py_name] = ScopeEntry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) - def c_name(self, name): def suggest_c_name(self, py_name): if self.prefix: return '{}_DOT_{}'.format(self.prefix, py_name) From 7cb7396f62532f96bed67ecb8829d8d27c8609d6 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 12 Dec 2017 13:18:22 +0100 Subject: [PATCH 61/66] Use suggest_c_name() not c_name() --- pyc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyc.py b/pyc.py index f4f4c261..dafcc57c 100755 --- a/pyc.py +++ b/pyc.py @@ -74,7 +74,7 @@ def compiler(self) -> str: def declare_var(self, node: ast.AnnAssign) -> str: py_name = node.target.id py_type = node.annotation.id - c_name = self.scope.c_name(py_name) + c_name = self.scope.suggest_c_name(py_name) if py_type == 'int': c_type = 'int32_t' def_value = '0' @@ -252,7 +252,7 @@ def compile(self) -> str: class ModuleCompiler(BaseCompiler): def compile(self) -> str: # Add a var for __name__ - dunder_name_c_name = self.scope.c_name('__name__') + dunder_name_c_name = self.scope.suggest_c_name('__name__') self.scope.add_entry( py_type='str', py_name='__name__', c_type='const char*', c_name=dunder_name_c_name) c_src = 'const char* {} = "{}";\n'.format(dunder_name_c_name, self.name.replace('.', '_DOT_')) From 8a7d99ded55d3d91e1d607e2fb2d0c095c6f1ad3 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 12 Dec 2017 13:19:19 +0100 Subject: [PATCH 62/66] Introduce BiLangScopeEntry --- pyc.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pyc.py b/pyc.py index dafcc57c..2404ec3f 100755 --- a/pyc.py +++ b/pyc.py @@ -9,7 +9,8 @@ LOG = logging.getLogger(__name__) -ScopeEntry = namedtuple('ScopeEntry', ['c_name', 'c_type', 'py_name', 'py_type']) +ScopeEntry = namedtuple('ScopeEntry', ['name', 'type', 'callable']) +BiLangScopeEntry = namedtuple('BiLangScopeEntry', ['c', 'py']) class Scope(dict): def __init__(self, parent=None, prefix=None): @@ -19,8 +20,8 @@ def __init__(self, parent=None, prefix=None): else: dict.__init__(self) - def add_entry(self, py_name, py_type, c_name, c_type): - self[py_name] = ScopeEntry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) + def add_entry(self, c, py): + self[py.name] = BiLangScopeEntry(c=c, py=py) def suggest_c_name(self, py_name): if self.prefix: @@ -83,15 +84,21 @@ def declare_var(self, node: ast.AnnAssign) -> str: def_value = 'NULL' else: raise NotImplementedError('unhandled py_type: {}'.format(py_type)) - self.scope.add_entry(py_name=py_name, py_type=py_type, c_name=c_name, c_type=c_type) - LOG.debug('set scope entry `%s` in compiler %s', py_name, self.name) + + # Register the new var in the scope + self.scope.add_entry( + py=ScopeEntry(name=py_name, type=py_type, callable=False), + c=ScopeEntry(name=c_name, type=c_type, callable=False), + ) + + LOG.debug('set scope entry `%s` in scope %s', py_name, self.name) return '{c_type} {c_name} = {def_value};'.format( c_type=c_type, c_name=c_name, def_value=def_value) def py_type(self, node): if type(node) in [ast.Name, ast.Attribute]: var = self.scope.resolve(node) - return var.py_type + return var.py.type else: raise NotImplementedError('cannot get type of {}'.format(ast.dump(node))) @@ -109,13 +116,13 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> str: raise CompileError('assignment to undeclared variable `{}` in scope {!r}'.format(py_name, self.scope)) decl = self.scope[py_name] - if decl.py_type == 'int': + if decl.py.type == 'int': if type(node.value) != ast.Num: raise CompileError( 'assignment of non-numerical value {} to int variable `{}`' .format(ast.dump(node), py_name)) value_src = self.visit(node.value) - elif decl.py_type == 'str': + elif decl.py.type == 'str': if type(node.value) != ast.Str: raise CompileError( 'assignment of non-string value {} to str variable `{}`' @@ -124,7 +131,7 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> str: else: raise NotImplementedError('unhandled py_type: {}'.format(decl.py_type)) return '{c_name} = {value_src}'.format( - c_name=decl.c_name, value_src=value_src) + c_name=decl.c.name, value_src=value_src) def visit_Name(self, node: ast.Name) -> str: py_name = node.id @@ -136,7 +143,7 @@ def visit_Name(self, node: ast.Name) -> str: if py_name not in self.scope: raise CompileError('NameError: undefined reference `{}`'.format(py_name)) - c_name = self.scope[py_name].c_name + c_name = self.scope[py_name].c.name LOG.debug('c_name(%r) == %s', py_name, c_name) return c_name @@ -254,7 +261,10 @@ def compile(self) -> str: # Add a var for __name__ dunder_name_c_name = self.scope.suggest_c_name('__name__') self.scope.add_entry( - py_type='str', py_name='__name__', c_type='const char*', c_name=dunder_name_c_name) + c=ScopeEntry(name=dunder_name_c_name, type='const char*', + callable=False), + py=ScopeEntry(name='__name__', type='str', callable=False), + ) c_src = 'const char* {} = "{}";\n'.format(dunder_name_c_name, self.name.replace('.', '_DOT_')) # Sort the body nodes by type (top-level code or functions) From 799d53bca17321a7d623f96ed75d7e7417d930c2 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Tue, 12 Dec 2017 17:27:24 +0100 Subject: [PATCH 63/66] Remove module self reference in scope --- pyc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyc.py b/pyc.py index 2404ec3f..21ece623 100755 --- a/pyc.py +++ b/pyc.py @@ -313,9 +313,6 @@ def compile(self) -> str: # The module exists in a new scope which inherits all builtin declarations main_scope = Scope(BUILTIN, prefix='MOD_{}'.format(module_name)) - - # In the __main__ module, __main__ is self-referential - main_scope[module_name] = main_scope # Create a new compiler for the __main__ module main_comp = ModuleCompiler(module_name, root, main_scope) From 2c5c7c2a9c465319c70ded7c05e13340ccfa2c56 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 13 Dec 2017 16:58:05 +0100 Subject: [PATCH 64/66] Check python version --- pyc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyc.py b/pyc.py index 21ece623..70c2378f 100755 --- a/pyc.py +++ b/pyc.py @@ -7,6 +7,8 @@ from collections import namedtuple +assert sys.version_info[:2] == (3, 6) + LOG = logging.getLogger(__name__) ScopeEntry = namedtuple('ScopeEntry', ['name', 'type', 'callable']) From 7566d77d23b5e05366749dbf197a49db3b1328b1 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 13 Dec 2017 21:14:17 +0100 Subject: [PATCH 65/66] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6e390aa3..6d2b1b37 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ test.c test.bin .vscode *.code-workspace +.DS_Store +build From 9b999f2c9860fc6278c8deaad38895e96a553ef2 Mon Sep 17 00:00:00 2001 From: Cyle Riggs Date: Wed, 13 Dec 2017 21:16:17 +0100 Subject: [PATCH 66/66] Add megadrive vdp.draw_text test --- pyc.py | 96 ++++++++++++++++++++++++------- run_tests.sh | 3 +- tests/if.comp.str/test.py | 8 +-- tests/return.constant.int/test.py | 4 +- tests/return.var.int/test.py | 4 +- tests/sega.vdp.draw_text/run.sh | 15 +++++ tests/sega.vdp.draw_text/test.py | 7 +++ 7 files changed, 106 insertions(+), 31 deletions(-) create mode 100644 tests/sega.vdp.draw_text/run.sh create mode 100644 tests/sega.vdp.draw_text/test.py diff --git a/pyc.py b/pyc.py index 70c2378f..7f1f9f33 100755 --- a/pyc.py +++ b/pyc.py @@ -54,6 +54,31 @@ def dict(self) -> dict: BUILTIN = Scope() +SegaScope = Scope() +SegaScope.add_entry( + c=ScopeEntry(name='VDP_init', type=None, callable=True), + py=ScopeEntry(name='init', type=None, callable=True) +) + +SegaScope.add_entry( + c=ScopeEntry(name='VDP_drawText', type=None, callable=True), + py=ScopeEntry(name='draw_text', type=None, callable=True) +) + +SysScope = Scope() +SysScope.add_entry( + c=ScopeEntry(name='exit', type=None, callable=True), + py=ScopeEntry(name='exit', type=None, callable=True), +) + +BUILTIN_MODS = { + 'sys': SysScope, + 'vdp': SegaScope, +} + +main_scope = Scope(BUILTIN, prefix='MOD___main__') + + class CompileError(RuntimeError): pass class BaseCompiler(ast.NodeVisitor): @@ -135,36 +160,61 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> str: return '{c_name} = {value_src}'.format( c_name=decl.c.name, value_src=value_src) + def _name_error(self, name): + raise CompileError('NameError: undefined reference `{}`'.format(name)) + def visit_Name(self, node: ast.Name) -> str: py_name = node.id - # Allow any reference into the C namespace. Let C compiler check them. - if py_name == 'C': - return '' - if py_name not in self.scope: - raise CompileError('NameError: undefined reference `{}`'.format(py_name)) + self._name_error(py_name) c_name = self.scope[py_name].c.name LOG.debug('c_name(%r) == %s', py_name, c_name) return c_name - + def visit_Attribute(self, node: ast.Attribute) -> str: - base_name = self.visit(node.value) - if base_name: - return '{}.{}'.format(self.visit(node.value), node.attr) + if node.value.id in BUILTIN_MODS: + attr_scope = BUILTIN_MODS[node.value.id] + if node.attr not in attr_scope: + self._name_error('{}.{}'.format(node.value.id, node.attr)) + return attr_scope[node.attr].c.name else: - return node.attr + self._name_error(node.value.id) def visit_Import(self, node: ast.Attribute) -> str: src = '' for alias in node.names: - # The 'C' module has an internal implementation - if alias.name == 'C': + # Skip modules with an internal implementation + if alias.name in BUILTIN_MODS: continue return src - def visit_If(self, node: ast.Attribute) -> str: + def visit_Pass(self, node: ast.Pass) -> str: + return '; // do nothing\n' + + def visit_NameConstant(self, node: ast.NameConstant) -> str: + if node.value == True: + return 'TRUE' + raise NotImplementedError(ast.dump(node)) + + def visit_While(self, node: ast.While) -> str: + # Compile the test condition + test_src = self.visit(node.test) + + # Compile the body + body_src = '' + for body_node in node.body: + line_name = '{}:{}'.format(body_node.lineno, body_node.col_offset) + line_comp = LineCompiler(line_name, body_node, self.scope) + body_src += line_comp.compile() + + if node.orelse: + raise NotImplementedError('while...else') + + return 'while ({test_src}) {{\n{body_src}\n}}'.format(test_src=test_src, body_src=body_src) + + def visit_If(self, node: ast.If) -> str: # Compile the test condition test_src = self.visit(node.test) @@ -298,24 +348,25 @@ def compile(self) -> str: class ProgramCompiler(object): - def __init__(self, name, py_src): + def __init__(self, name, py_src, platform): + self.platform = platform self.name = name self.py_src = py_src def _pre_source(self) -> str: - return '\n'.join([ - '#include ', - '#include ', - '#include ',]) + '\n\n' + if self.platform == 'unix': + return '\n'.join([ + '#include ', + '#include ', + '#include ',]) + '\n\n' + elif self.platform == 'md': + return '#include \n\n' def compile(self) -> str: # Use CPython's builtin source parser root = ast.parse(self.py_src) module_name = '__main__' - # The module exists in a new scope which inherits all builtin declarations - main_scope = Scope(BUILTIN, prefix='MOD_{}'.format(module_name)) - # Create a new compiler for the __main__ module main_comp = ModuleCompiler(module_name, root, main_scope) @@ -331,6 +382,7 @@ def compile(self) -> str: def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) p.add_argument('sourcefile', type=argparse.FileType('r')) + p.add_argument('--platform', '-p', choices=['md', 'unix'], default='sega') return p.parse_args() @@ -339,7 +391,7 @@ def main() -> int: args = parse_args() py_src = args.sourcefile.read() module_name = os.path.basename(args.sourcefile.name) - prog_compiler = ProgramCompiler(module_name, py_src) + prog_compiler = ProgramCompiler(module_name, py_src, args.platform) print(prog_compiler.compile()) return os.EX_OK diff --git a/run_tests.sh b/run_tests.sh index af26d9a0..34c07113 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,8 @@ #!/bin/bash export PYTHON=python3.6 export GCC="gcc -Wall -pedantic-errors" -export PYC="$(pwd)/pyc.py" +export PYC="$(pwd)/pyc.py -p unix" +export PYC_MD="$(pwd)/pyc.py -p md" FAILED=0 PASSED=0 diff --git a/tests/if.comp.str/test.py b/tests/if.comp.str/test.py index c8fd9fcc..6e525580 100644 --- a/tests/if.comp.str/test.py +++ b/tests/if.comp.str/test.py @@ -1,12 +1,12 @@ -import C +import sys foo: str = 'foo' bar: str = 'bar' if __name__ == '__main__': if foo != 'foo': - C.exit(1) + sys.exit(1) if foo == bar: - C.exit(2) + sys.exit(2) - C.exit(42) + sys.exit(42) diff --git a/tests/return.constant.int/test.py b/tests/return.constant.int/test.py index ea6cdf89..0e2f6d37 100644 --- a/tests/return.constant.int/test.py +++ b/tests/return.constant.int/test.py @@ -1,4 +1,4 @@ -import C +import sys if __name__ == '__main__': - C.exit(42) + sys.exit(42) diff --git a/tests/return.var.int/test.py b/tests/return.var.int/test.py index 463300fc..af6586c3 100644 --- a/tests/return.var.int/test.py +++ b/tests/return.var.int/test.py @@ -1,4 +1,4 @@ -import C +import sys if __name__ == '__main__': x: int = 42 - C.exit(x) + sys.exit(x) diff --git a/tests/sega.vdp.draw_text/run.sh b/tests/sega.vdp.draw_text/run.sh new file mode 100644 index 00000000..3622f691 --- /dev/null +++ b/tests/sega.vdp.draw_text/run.sh @@ -0,0 +1,15 @@ +#!/bin/sh +${PYC_MD} test.py > test.c +if [ $? -ne 0 ]; then + exit 1 +fi + +export GCC_MD="docker run -v $(pwd)/build:/src --rm -it beardedfoo/gendev:0.3.0" +mkdir -p build +cp *.c build/ +pushd build +${GCC_MD} +if [ $? -eq 0 ]; then + exit 42 +fi +exit $? diff --git a/tests/sega.vdp.draw_text/test.py b/tests/sega.vdp.draw_text/test.py new file mode 100644 index 00000000..960aaf36 --- /dev/null +++ b/tests/sega.vdp.draw_text/test.py @@ -0,0 +1,7 @@ +# import vdp +vdp.init() +vdp.draw_text("Hello World for Sega Megadrive", 10, 13) +vdp.draw_text("By @beardedfoo", 10, 15) +vdp.draw_text("...in Python!", 10, 17) +while True: + pass