type Type = py.Type

type Map<T> = pxt.Map<T>;

import * as nodeutil from './nodeutil';

import * as fs from 'fs';

import U = pxt.Util;

import B = pxt.blocks;

namespace py {

export interface TypeOptions {

union?: Type;

classType?: py.ClassDef;

primType?: string;

arrayType?: Type;

}

export interface Type extends TypeOptions {

tid: number;

}

export interface FieldDesc {

name: string;

type: Type;

inClass: py.ClassDef;

fundef?: py.FunctionDef;

isGetSet?: boolean;

isStatic?: boolean;

isProtected?: boolean;

initializer?: py.Expr;

}

export interface VarDescOptions {

expandsTo?: string;

isImportStar?: boolean;

isPlainImport?: boolean;

isLocal?: boolean;

isParam?: boolean;

fundef?: py.FunctionDef;

classdef?: py.ClassDef;

isImport?: py.Module;

}

export interface VarDesc extends VarDescOptions {

type: Type;

name: string;

}

// based on grammar at https://docs.python.org/3/library/ast.html

export interface AST {

lineno: number;

col_offset: number;

startPos?: number;

endPos?: number;

kind: string;

}

export interface Stmt extends AST {

_stmtBrand: void;

}

export interface Symbol extends Stmt {

_symbolBrand: void;

}

export interface Expr extends AST {

tsType?: Type;

_exprBrand: void;

}

export type expr_context = "Load" | "Store" | "Del" | "AugLoad" | "AugStore" | "Param"

export type boolop = "And" | "Or"

export type operator = "Add" | "Sub" | "Mult" | "MatMult" | "Div" | "Mod" | "Pow"

| "LShift" | "RShift" | "BitOr" | "BitXor" | "BitAnd" | "FloorDiv"

export type unaryop = "Invert" | "Not" | "UAdd" | "USub"

export type cmpop = "Eq" | "NotEq" | "Lt" | "LtE" | "Gt" | "GtE" | "Is" | "IsNot" | "In" | "NotIn"

export type identifier = string

export type int = number

export interface Arg extends AST {

kind: "Arg";

arg: identifier;

annotation?: Expr;

type?: Type;

}

export interface Arguments extends AST {

kind: "Arguments";

args: Arg[];

vararg?: Arg;

kwonlyargs: Arg[];

kw_defaults: Expr[];

kwarg?: Arg;

defaults: Expr[];

}

// keyword arguments supplied to call (NULL identifier for **kwargs)

export interface Keyword extends AST {

kind: "Keyword";

arg?: identifier;

value: Expr;

}

export interface Comprehension extends AST {

kind: "Comprehension";

target: Expr;

iter: Expr;

ifs: Expr[];

is_async: int;

}

export interface Module extends Symbol, ScopeDef {

kind: "Module";

body: Stmt[];

name?: string;

source?: string[];

comments?: string[];

}

export interface ExceptHandler extends AST {

kind: "ExceptHandler";

type?: Expr;

name?: identifier;

body: Stmt[];

}

// import name with optional 'as' alias.

export interface Alias extends AST {

kind: "Alias";

name: identifier;

asname?: identifier;

}

export interface WithItem extends AST {

kind: "WithItem";

context_expr: Expr;

optional_vars?: Expr;

}

export interface AnySlice extends AST {

_anySliceBrand: void;

}

export interface Slice extends AnySlice {

kind: "Slice";

lower?: Expr;

upper?: Expr;

step?: Expr;

}

export interface ExtSlice extends AnySlice {

kind: "ExtSlice";

dims: AnySlice[];

}

export interface Index extends AnySlice {

kind: "Index";

value: Expr;

}

export interface ScopeDef extends Stmt {

vars?: Map<VarDesc>;

parent?: ScopeDef;

}

export interface FunctionDef extends Symbol, ScopeDef {

kind: "FunctionDef";

name: identifier;

args: Arguments;

body: Stmt[];

decorator_list: Expr[];

returns?: Expr;

retType?: Type;

alwaysThrows?: boolean;

}

export interface AsyncFunctionDef extends Stmt {

kind: "AsyncFunctionDef";

name: identifier;

args: Arguments;

body: Stmt[];

decorator_list: Expr[];

returns?: Expr;

}

export interface ClassDef extends Symbol, ScopeDef {

kind: "ClassDef";

name: identifier;

bases: Expr[];

keywords: Keyword[];

body: Stmt[];

decorator_list: Expr[];

fields?: Map<FieldDesc>;

baseClass?: ClassDef;

}

export interface Return extends Stmt {

kind: "Return";

value?: Expr;

}

export interface Delete extends Stmt {

kind: "Delete";

targets: Expr[];

}

export interface Assign extends Symbol {

kind: "Assign";

targets: Expr[];

value: Expr;

}

export interface AugAssign extends Stmt {

kind: "AugAssign";

target: Expr;

op: operator;

value: Expr;

}

export interface AnnAssign extends Stmt {

kind: "AnnAssign";

target: Expr;

annotation: Expr;

value?: Expr;

simple: int; // 'simple' indicates that we annotate simple name without parens

}

export interface For extends Stmt {

kind: "For";

target: Expr;

iter: Expr;

body: Stmt[];

orelse: Stmt[]; // use 'orelse' because else is a keyword in target languages

}

export interface AsyncFor extends Stmt {

kind: "AsyncFor";

target: Expr;

iter: Expr;

body: Stmt[];

orelse: Stmt[];

}

export interface While extends Stmt {

kind: "While";

test: Expr;

body: Stmt[];

orelse: Stmt[];

}

export interface If extends Stmt {

kind: "If";

test: Expr;

body: Stmt[];

orelse: Stmt[];

}

export interface With extends Stmt {

kind: "With";

items: WithItem[];

body: Stmt[];

}

export interface AsyncWith extends Stmt {

kind: "AsyncWith";

items: WithItem[];

body: Stmt[];

}

export interface Raise extends Stmt {

kind: "Raise";

exc?: Expr;

cause?: Expr;

}

export interface Try extends Stmt {

kind: "Try";

body: Stmt[];

handlers: ExceptHandler[];

orelse: Stmt[];

finalbody: Stmt[];

}

export interface Assert extends Stmt {

kind: "Assert";

test: Expr;

msg?: Expr;

}

export interface Import extends Stmt {

kind: "Import";

names: Alias[];

}

export interface ImportFrom extends Stmt {

kind: "ImportFrom";

module?: identifier;

names: Alias[];

level?: int;

}

export interface Global extends Stmt {

kind: "Global";

names: identifier[];

}

export interface Nonlocal extends Stmt {

kind: "Nonlocal";

names: identifier[];

}

export interface ExprStmt extends Stmt {

kind: "ExprStmt";

value: Expr;

}

export interface Pass extends Stmt {

kind: "Pass";

}

export interface Break extends Stmt {

kind: "Break";

}

export interface Continue extends Stmt {

kind: "Continue";

}

export interface BoolOp extends Expr {

kind: "BoolOp";

op: boolop;

values: Expr[];

}

export interface BinOp extends Expr {

kind: "BinOp";

left: Expr;

op: operator;

right: Expr;

}

export interface UnaryOp extends Expr {

kind: "UnaryOp";

op: unaryop;

operand: Expr;

}

export interface Lambda extends Expr {

kind: "Lambda";

args: Arguments;

body: Expr;

}

export interface IfExp extends Expr {

kind: "IfExp";

test: Expr;

body: Expr;

orelse: Expr;

}

export interface Dict extends Expr {

kind: "Dict";

keys: Expr[];

values: Expr[];

}

export interface Set extends Expr {

kind: "Set";

elts: Expr[];

}

export interface ListComp extends Expr {

kind: "ListComp";

elt: Expr;

generators: Comprehension[];

}

export interface SetComp extends Expr {

kind: "SetComp";

elt: Expr;

generators: Comprehension[];

}

export interface DictComp extends Expr {

kind: "DictComp";

key: Expr;

value: Expr;

generators: Comprehension[];

}

export interface GeneratorExp extends Expr {

kind: "GeneratorExp";

elt: Expr;

generators: Comprehension[];

}

export interface Await extends Expr {

kind: "Await";

value: Expr;

}

export interface Yield extends Expr {

kind: "Yield";

value?: Expr;

}

export interface YieldFrom extends Expr {

kind: "YieldFrom";

value: Expr;

}

// need sequences for compare to distinguish between x < 4 < 3 and (x < 4) < 3

export interface Compare extends Expr {

kind: "Compare";

left: Expr;

ops: cmpop[];

comparators: Expr[];

}

export interface Call extends Expr {

kind: "Call";

func: Expr;

args: Expr[];

keywords: Keyword[];

}

export interface Num extends Expr {

kind: "Num";

n: number;

ns: string;

}

export interface Str extends Expr {

kind: "Str";

s: string;

}

export interface FormattedValue extends Expr {

kind: "FormattedValue";

value: Expr;

conversion?: int;

format_spec?: Expr;

}

export interface JoinedStr extends Expr {

kind: "JoinedStr";

values: Expr[];

}

export interface Bytes extends Expr {

kind: "Bytes";

s: number[];

}

export interface NameConstant extends Expr {

kind: "NameConstant";

value: boolean; // null=None, True, False

}

export interface Ellipsis extends Expr {

kind: "Ellipsis";

}

export interface Constant extends Expr {

kind: "Constant";

value: any; // ???

}

// the following expression can appear in assignment context

export interface AssignmentExpr extends Expr { }

export interface Attribute extends AssignmentExpr {

kind: "Attribute";

value: Expr;

attr: identifier;

ctx: expr_context;

}

export interface Subscript extends AssignmentExpr {

kind: "Subscript";

value: Expr;

slice: AnySlice;

ctx: expr_context;

}

export interface Starred extends AssignmentExpr {

kind: "Starred";

value: Expr;

ctx: expr_context;

}

export interface Name extends AssignmentExpr {

kind: "Name";

id: identifier;

ctx: expr_context;

isdef?: boolean;

}

export interface List extends AssignmentExpr {

kind: "List";

elts: Expr[];

ctx: expr_context;

}

export interface Tuple extends AssignmentExpr {

kind: "Tuple";

elts: Expr[];

ctx: expr_context;

}

}

/* tslint:disable:no-trailing-whitespace */

const convPy = `

import ast

import sys

import json

def to_json(val):

if val is None or isinstance(val, (bool, str, int, float)):

return val

if isinstance(val, list):

return [to_json(x) for x in val]

if isinstance(val, ast.AST):

js = dict()

js['kind'] = val.__class__.__name__

for attr_name in dir(val):

if not attr_name.startswith("_"):

js[attr_name] = to_json(getattr(val, attr_name))

return js

if isinstance(val, (bytearray, bytes)):

return [x for x in val]

raise Exception("unhandled: %s (type %s)" % (val, type(val)))

js = dict()

for fn in @files@:

js[fn] = to_json(ast.parse(open(fn, "r").read()))

print(json.dumps(js))

`

/* tslint:enable:no-trailing-whitespace */

const nameMap: Map<string> = {

"Expr": "ExprStmt",

"arg": "Arg",

"arguments": "Arguments",

"keyword": "Keyword",

"comprehension": "Comprehension",

"alias": "Alias",

"withitem": "WithItem"

}

const simpleNames: Map<boolean> = {

"Load": true, "Store": true, "Del": true, "AugLoad": true, "AugStore": true, "Param": true, "And": true,

"Or": true, "Add": true, "Sub": true, "Mult": true, "MatMult": true, "Div": true, "Mod": true, "Pow": true,

"LShift": true, "RShift": true, "BitOr": true, "BitXor": true, "BitAnd": true, "FloorDiv": true,

"Invert": true, "Not": true, "UAdd": true, "USub": true, "Eq": true, "NotEq": true, "Lt": true, "LtE": true,

"Gt": true, "GtE": true, "Is": true, "IsNot": true, "In": true, "NotIn": true,

}

function stmtTODO(v: py.Stmt) {

pxt.tickEvent("python.todo", { kind: v.kind })

return B.mkStmt(B.mkText("TODO: " + v.kind))

}

function exprTODO(v: py.Expr) {

pxt.tickEvent("python.todo", { kind: v.kind })

return B.mkText(" {TODO: " + v.kind + "} ")

}

function docComment(cmt: string) {

if (cmt.trim().split(/\n/).length <= 1)

cmt = cmt.trim()

else

cmt = cmt + "\n"

return B.mkStmt(B.mkText("/** " + cmt + " */"))

}

let moduleAst: Map<py.Module> = {}

function lookupSymbol(name: string): py.Symbol {

if (!name) return null

if (moduleAst[name])

return moduleAst[name]

let parts = name.split(".")

if (parts.length >= 2) {

let last = parts.length - 1

let par = moduleAst[parts.slice(0, last).join(".")]

let ename = parts[last]

if (par) {

for (let stmt of par.body) {

if (stmt.kind == "ClassDef" || stmt.kind == "FunctionDef") {

if ((stmt as py.FunctionDef).name == ename)

return stmt as py.FunctionDef

}

if (stmt.kind == "Assign") {

let assignment = stmt as py.Assign

if (assignment.targets.length == 1 && getName(assignment.targets[0]) == ename) {

return assignment

}

}

}

}

}

return null

}

interface Ctx {

currModule: py.Module;

currClass: py.ClassDef;

currFun: py.FunctionDef;

}

let ctx: Ctx

let currIteration = 0

let typeId = 0

let numUnifies = 0

function mkType(o: py.TypeOptions = {}) {

let r: Type = U.flatClone(o) as any

r.tid = ++typeId

return r

}

function currentScope(): py.ScopeDef {

return ctx.currFun || ctx.currClass || ctx.currModule

}

function defvar(n: string, opts: py.VarDescOptions) {

let scopeDef = currentScope()

let v = scopeDef.vars[n]

if (!v) {

v = scopeDef.vars[n] = { type: mkType(), name: n }

}

for (let k of Object.keys(opts)) {

(v as any)[k] = (opts as any)[k]

}

return v

}

let tpString: Type = mkType({ primType: "string" })

let tpNumber: Type = mkType({ primType: "number" })

let tpBoolean: Type = mkType({ primType: "boolean" })

let tpBuffer: Type = mkType({ primType: "Buffer" })

let tpVoid: Type = mkType({ primType: "void" })

function find(t: Type) {

if (t.union) {

t.union = find(t.union)

return t.union

}

return t

}

function getFullName(n: py.AST): string {

let s = n as py.ScopeDef

let pref = ""

if (s.parent) {

pref = getFullName(s.parent)

if (!pref) pref = ""

else pref += "."

}

let nn = n as py.FunctionDef

if (nn.name) return pref + nn.name

else return pref + "?" + n.kind

}

function applyTypeMap(s: string) {

let over = U.lookup(typeMap, s)

if (over) return over

for (let v of U.values(ctx.currModule.vars)) {

if (!v.isImport)

continue

if (v.expandsTo == s) return v.name

if (v.isImport && U.startsWith(s, v.expandsTo + ".")) {

return v.name + s.slice(v.expandsTo.length)

}

}

return s

}

function t2s(t: Type): string {

t = find(t)

if (t.primType)

return t.primType

else if (t.classType)

return applyTypeMap(getFullName(t.classType))

else if (t.arrayType)

return t2s(t.arrayType) + "[]"

else

return "?" + t.tid

}

let currErrs = ""

function error(t0: Type, t1: Type) {

currErrs += "types not compatible: " + t2s(t0) + " and " + t2s(t1) + "; "

}

function typeCtor(t: Type): any {

if (t.primType) return t.primType

else if (t.classType) return t.classType

else if (t.arrayType) return "array"

return null

}

function isFree(t: Type) {

return !typeCtor(find(t))

}

function canUnify(t0: Type, t1: Type): boolean {

t0 = find(t0)

t1 = find(t1)

if (t0 === t1)

return true

let c0 = typeCtor(t0)

let c1 = typeCtor(t1)

if (!c0 || !c1)

return true

if (c0 !== c1)

return false

if (c0 == "array") {

return canUnify(t0.arrayType, t1.arrayType)

}

return true

}

function unifyClass(t: Type, cd: py.ClassDef) {

t = find(t)

if (t.classType == cd) return

if (isFree(t)) {

t.classType = cd

return

}

unify(t, mkType({ classType: cd }))

}

function unify(t0: Type, t1: Type): void {

t0 = find(t0)

t1 = find(t1)

if (t0 === t1)

return

if (!canUnify(t0, t1)) {

error(t0, t1)

return

}

if (typeCtor(t0) && !typeCtor(t1))

return unify(t1, t0)

numUnifies++

t0.union = t1

if (t0.arrayType && t1.arrayType)

unify(t0.arrayType, t1.arrayType)

}

function getClassField(ct: py.ClassDef, n: string, checkOnly = false, skipBases = false) {

if (!ct.fields)

ct.fields = {}

if (!ct.fields[n]) {

if (!skipBases)

for (let par = ct.baseClass; par; par = par.baseClass) {

if (par.fields && par.fields[n])

return par.fields[n]

}

if (checkOnly) return null

ct.fields[n] = {

inClass: ct,

name: n,

type: mkType()

}

}

return ct.fields[n]

}

function getTypeField(t: Type, n: string, checkOnly = false) {

t = find(t)

let ct = t.classType

if (ct)

return getClassField(ct, n, checkOnly)

return null

}

function lookupVar(n: string) {

let s = currentScope()

while (s) {

let v = U.lookup(s.vars, n)

if (v) return v

// go to parent, excluding class scopes

do {

s = s.parent

} while (s && s.kind == "ClassDef")

}

return null

}

function getClassDef(e: py.Expr) {

let n = getName(e)

let v = lookupVar(n)

if (v)

return v.classdef

let s = lookupSymbol(n)

if (s && s.kind == "ClassDef")

return s as py.ClassDef

return null

}

function typeOf(e: py.Expr): Type {

if (e.tsType) {

return find(e.tsType)

} else {

e.tsType = mkType()

return e.tsType

}

}

function isOfType(e: py.Expr, name: string) {

let t = typeOf(e)

if (t.classType && t.classType.name == name)

return true

if (t2s(t) == name)

return true

return false

}

function resetCtx(m: py.Module) {

ctx = {

currClass: null,

currFun: null,

currModule: m

}

}

function scope(f: () => B.JsNode) {

const prevCtx = U.flatClone(ctx)

let r: B.JsNode;

try {

r = f()

} finally {

ctx = prevCtx

}

return r;

}

function todoExpr(name: string, e: B.JsNode) {

if (!e)

return B.mkText("")

return B.mkGroup([B.mkText("/* TODO: " + name + " "), e, B.mkText(" */")])

}

function todoComment(name: string, n: B.JsNode[]) {

if (n.length == 0)

return B.mkText("")

return B.mkGroup([B.mkText("/* TODO: " + name + " "), B.mkGroup(n), B.mkText(" */"), B.mkNewLine()])

}

function doKeyword(k: py.Keyword) {

let t = expr(k.value)

if (k.arg)

return B.mkInfix(B.mkText(k.arg), "=", t)

else

return B.mkGroup([B.mkText("**"), t])

}

function doArgs(args: py.Arguments, isMethod: boolean) {

U.assert(!args.kwonlyargs.length)

let nargs = args.args.slice()

if (isMethod) {

U.assert(nargs[0].arg == "self")

nargs.shift()

} else {

U.assert(!nargs[0] || nargs[0].arg != "self")

}

let didx = args.defaults.length - nargs.length

let lst = nargs.map(a => {

let v = defvar(a.arg, { isParam: true })

if (!a.type) a.type = v.type

let res = [quote(a.arg), typeAnnot(v.type)]

if (a.annotation)

res.push(todoExpr("annotation", expr(a.annotation)))

if (didx >= 0) {

res.push(B.mkText(" = "))

res.push(expr(args.defaults[didx]))

unify(a.type, typeOf(args.defaults[didx]))

}

didx++

return B.mkGroup(res)

})

if (args.vararg)

lst.push(B.mkText("TODO *" + args.vararg.arg))

if (args.kwarg)

lst.push(B.mkText("TODO **" + args.kwarg.arg))

return B.H.mkParenthesizedExpression(B.mkCommaSep(lst))

}

const numOps: Map<number> = {

Sub: 1,

Div: 1,

Pow: 1,

LShift: 1,

RShift: 1,

BitOr: 1,

BitXor: 1,

BitAnd: 1,

FloorDiv: 1,

Mult: 1, // this can be also used on strings and arrays, but let's ignore that for now

}

const opMapping: Map<string> = {

Add: "+",

Sub: "-",

Mult: "*",

MatMult: "Math.matrixMult",

Div: "/",

Mod: "%",

Pow: "**",

LShift: "<<",

RShift: ">>",

BitOr: "|",

BitXor: "^",

BitAnd: "&",

FloorDiv: "Math.idiv",

And: "&&",

Or: "||",

Eq: "==",

NotEq: "!=",

Lt: "<",

LtE: "<=",

Gt: ">",

GtE: ">=",

Is: "===",

IsNot: "!==",

In: "py.In",

NotIn: "py.NotIn",

}

const prefixOps: Map<string> = {

Invert: "~",

Not: "!",

UAdd: "P+",

USub: "P-",

}

const typeMap: pxt.Map<string> = {

"adafruit_bus_device.i2c_device.I2CDevice": "pins.I2CDevice"

}

function stmts(ss: py.Stmt[]) {

return B.mkBlock(ss.map(stmt))

}

function exprs0(ee: py.Expr[]) {

ee = ee.filter(e => !!e)

return ee.map(expr)

}

function setupScope(n: py.ScopeDef) {

if (!n.vars) {

n.vars = {}

n.parent = currentScope()

}

}

function typeAnnot(t: Type) {

let s = t2s(t)

if (s[0] == "?")

return B.mkText(": any; /** TODO: type **/")

return B.mkText(": " + t2s(t))

}

function guardedScope(v: py.AST, f: () => B.JsNode) {

try {

return scope(f);

}

catch (e) {

return B.mkStmt(todoComment(`conversion failed for ${(v as any).name || v.kind}`, []));

}

}

const stmtMap: Map<(v: py.Stmt) => B.JsNode> = {

FunctionDef: (n: py.FunctionDef) => guardedScope(n, () => {

let isMethod = !!ctx.currClass && !ctx.currFun

if (!isMethod)

defvar(n.name, { fundef: n })

setupScope(n)

ctx.currFun = n

if (!n.retType) n.retType = mkType()

let prefix = ""

let funname = n.name

let decs = n.decorator_list.filter(d => {

if (getName(d) == "property") {

prefix = "get"

return false

}

if (d.kind == "Attribute" && (d as py.Attribute).attr == "setter" &&

(d as py.Attribute).value.kind == "Name") {

funname = ((d as py.Attribute).value as py.Name).id

prefix = "set"

return false

}

return true

})

let nodes = [

todoComment("decorators", decs.map(expr))

]

if (isMethod) {

let fd = getClassField(ctx.currClass, funname, false, true)

if (n.body.length == 1 && n.body[0].kind == "Raise")

n.alwaysThrows = true

if (n.name == "__init__") {

nodes.push(B.mkText("constructor"))

unifyClass(n.retType, ctx.currClass)

} else {

if (funname == "__get__" || funname == "__set__") {

let i2cArg = "i2cDev"

let vv = n.vars[i2cArg]

if (vv) {

let i2cDevClass =

lookupSymbol("adafruit_bus_device.i2c_device.I2CDevice") as py.ClassDef

if (i2cDevClass)

unifyClass(vv.type, i2cDevClass)

}

vv = n.vars["value"]

if (funname == "__set__" && vv) {

let cf = getClassField(ctx.currClass, "__get__")

if (cf.fundef)

unify(vv.type, cf.fundef.retType)

}

let nargs = n.args.args

if (nargs[1].arg == "obj") {

// rewrite

nargs[1].arg = i2cArg

if (nargs[nargs.length - 1].arg == "objtype") {

nargs.pop()

n.args.defaults.pop()

}

iterPy(n, e => {

if (e.kind == "Attribute") {

let a = e as py.Attribute

if (a.attr == "i2c_device" && getName(a.value) == "obj") {

let nm = e as py.Name

nm.kind = "Name"

nm.id = i2cArg

delete a.attr

delete a.value

}

}

})

}