From 0ad212f16318a943351c001f8392bce03d8a3403 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 7 Aug 2026 14:20:12 -0700 Subject: [PATCH 1/3] [SQL] ExpressionBuilder helper for inner IR unit tests Signed-off-by: Mihai Budiu --- .../visitors/inner/ResolveReferences.java | 1 + .../compiler/ir/DetectShuffleTests.java | 29 +-- .../compiler/ir/EquivalenceTests.java | 202 ++++++------------ .../compiler/ir/InliningTests.java | 68 +++--- .../sqlCompiler/compiler/ir/InternTest.java | 44 ++-- .../compiler/ir/TestSimplifyConditionals.java | 83 ++----- .../compiler/ir/UnusedFieldsTest.java | 49 ++--- .../compiler/sql/tools/ExpressionBuilder.java | 171 +++++++++++++++ .../inner/ResolveReferencesTests.java | 133 ++++++++++++ 9 files changed, 467 insertions(+), 313 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/ExpressionBuilder.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferencesTests.java diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java index dbab98888d5..716166da9a7 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferences.java @@ -73,6 +73,7 @@ public VisitDecision preorder(DBSPLetExpression expression) { expression.initializer.accept(this); this.substitutionContext.newContext(); this.substitutionContext.substitute(expression.variable.variable, expression); + this.reference.declare(expression.variable, expression); expression.consumer.accept(this); this.substitutionContext.popContext(); return VisitDecision.STOP; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/DetectShuffleTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/DetectShuffleTests.java index 5e5ef7ef584..65329536a12 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/DetectShuffleTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/DetectShuffleTests.java @@ -3,64 +3,57 @@ import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.inner.DetectShuffle; import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; -import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.junit.Assert; import org.junit.Test; /** Tests for the DetectShuffle visitor */ public class DetectShuffleTests { + final ExpressionBuilder b = new ExpressionBuilder(); + @Test public void detectShuffle() { final DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - final var tuple = new DBSPTypeTuple( + final var tuple = b.tup( DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT64, false), DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, false), - new DBSPTypeTuple(DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT16, true)) - ); - final var var = tuple.ref().var(); + b.tup(DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT16, true))); + final var var = b.refVar(tuple); // simple shuffle final DBSPClosureExpression clo0 = - new DBSPTupleExpression(var.deref().field(0), var.deref().field(2)) - .closure(var); + b.tuple(b.field(var, 0), b.field(var, 2)).closure(var); var shuffle = DetectShuffle.analyze(compiler, clo0); Assert.assertNotNull(shuffle); Assert.assertEquals("[0, 2]", shuffle.toString()); // clone() final DBSPClosureExpression clo1 = - new DBSPTupleExpression(var.deref().field(0).applyClone(), - var.deref().field(1)).closure(var); + b.tuple(b.field(var, 0).applyClone(), b.field(var, 1)).closure(var); shuffle = DetectShuffle.analyze(compiler, clo1); Assert.assertNotNull(shuffle); Assert.assertEquals("[0, 1]", shuffle.toString()); // repeated fields final DBSPClosureExpression clo2 = - new DBSPTupleExpression(var.deref().field(2), var.deref().field(2)) - .closure(var); + b.tuple(b.field(var, 2), b.field(var, 2)).closure(var); shuffle = DetectShuffle.analyze(compiler, clo2); Assert.assertNotNull(shuffle); Assert.assertEquals("[2, 2]", shuffle.toString()); // not a shuffle: constant field final DBSPClosureExpression clo3 = - new DBSPTupleExpression(var.deref().field(2), new DBSPI32Literal(10)) - .closure(var); + b.tuple(b.field(var, 2), b.lit(10)).closure(var); shuffle = DetectShuffle.analyze(compiler, clo3); Assert.assertNull(shuffle); // not a shuffle: nested deref final DBSPClosureExpression clo4 = - new DBSPTupleExpression( - var.deref().field(2).field(0), var.deref().field(1)) - .closure(var); + b.tuple(b.field(var, 2).field(0), b.field(var, 1)).closure(var); shuffle = DetectShuffle.analyze(compiler, clo4); Assert.assertNull(shuffle); } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/EquivalenceTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/EquivalenceTests.java index e3e4259a6b4..f5d78957db2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/EquivalenceTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/EquivalenceTests.java @@ -6,70 +6,55 @@ import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteEmptyRel; -import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; import org.dbsp.sqlCompiler.compiler.visitors.inner.ExpressionsCSE; import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; import org.dbsp.sqlCompiler.compiler.visitors.inner.ResolveReferences; import org.dbsp.sqlCompiler.compiler.visitors.inner.ValueNumbering; import org.dbsp.sqlCompiler.ir.IDBSPInnerNode; -import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPUnaryExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; import org.dbsp.sqlCompiler.ir.expression.DBSPZSetExpression; import org.dbsp.sqlCompiler.ir.expression.literal.DBSPBoolLiteral; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPLiteral; import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.dbsp.util.Linq; import org.junit.Assert; import org.junit.Test; -/** Unit tests for expression equivalence */ +/** Unit tests for expression equivalence. + * The alpha-equivalence tests build variables by hand: they need specific + * names shared between distinct nodes, which {@link ExpressionBuilder} + * avoids by construction. */ public class EquivalenceTests { - static DBSPExpression neg(DBSPExpression expression) { - return new DBSPUnaryExpression( - expression.getNode(), expression.getType(), DBSPOpcode.NEG, expression); - } + final ExpressionBuilder b = new ExpressionBuilder(); - static DBSPExpression binary(DBSPOpcode opcode, DBSPExpression left, DBSPExpression right) { - return new DBSPBinaryExpression( - left.getNode(), left.getType(), opcode, left, right); - } + /** Count the let expressions that CSE introduced */ + static void assertLets(DBSPCompiler compiler, IDBSPInnerNode node, int expected) { + InnerVisitor visitor = new InnerVisitor(compiler) { + int lets = 0; - static DBSPExpression add(DBSPExpression left, DBSPExpression right) { - return binary(DBSPOpcode.ADD, left, right); - } + @Override + public void postorder(DBSPLetExpression expression) { + this.lets++; + } - @Test - public void testCSE() { - DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i = new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); - DBSPTypeTuple tuple = new DBSPTypeTuple(i, i, i); - DBSPVariablePath var = tuple.ref().var(); - DBSPExpression c = new DBSPI32Literal(1); - DBSPExpression v0 = var.deref().field(0); - DBSPExpression v1 = var.deref().field(1); - DBSPExpression v2 = var.deref().field(2); - DBSPExpression v0m = neg(v0); - DBSPExpression p0 = binary(DBSPOpcode.DIV, v0m, v1); - DBSPExpression a = binary(DBSPOpcode.MUL, p0, p0); - DBSPExpression b = add(v2, c); - DBSPExpression d = add(b, b); - DBSPExpression body = new DBSPTupleExpression(a, d); - DBSPClosureExpression closure = body.closure(var.asParameter()); + @Override + public void endVisit() { + Assert.assertEquals(expected, this.lets); + } + }; + visitor.apply(node); + } + static IDBSPInnerNode cse(DBSPCompiler compiler, DBSPClosureExpression closure) { DBSPOperator fake = new DBSPConstantOperator( CalciteEmptyRel.INSTANCE, new DBSPZSetExpression(new DBSPBoolLiteral()), false); ValueNumbering numbering = new ValueNumbering(compiler); @@ -78,30 +63,27 @@ public void testCSE() { ExpressionsCSE cse = new ExpressionsCSE(compiler, numbering.canonical); cse.setOperatorContext(fake); cse.apply(closure); - IDBSPInnerNode translated = cse.get(closure); - InnerVisitor visitor = new InnerVisitor(compiler) { - int vars = 0; - public void postorder(DBSPLetExpression expression) { - this.vars++; - } + return cse.get(closure); + } - @Override - public void endVisit() { - // Find 2 common subexpressions; one has a single use - Assert.assertEquals(2, this.vars); - } - }; - visitor.apply(translated); + @Test + public void testCSE() { + DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); + DBSPClosureExpression closure = b.closure(b.tup(b.i32n(), b.i32n(), b.i32n()), t -> { + DBSPExpression p0 = b.binary(DBSPOpcode.DIV, b.neg(b.field(t, 0)), b.field(t, 1)); + DBSPExpression a = b.binary(DBSPOpcode.MUL, p0, p0); + DBSPExpression sum = b.add(b.field(t, 2), b.lit(1)); + return b.tuple(a, b.add(sum, sum)); + }); + // Find 2 common subexpressions; one has a single use + assertLets(compiler, cse(compiler, closure), 2); } @Test public void testLetEquivalenceContextUnchanged() { // Comparing let expressions may not modify the caller's context - DBSPType i = new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); - DBSPVariablePath v0 = i.var(); - DBSPVariablePath v1 = i.var(); - DBSPExpression let0 = new DBSPLetExpression(v0, new DBSPI32Literal(2, true), add(v0, v0)); - DBSPExpression let1 = new DBSPLetExpression(v1, new DBSPI32Literal(2, true), add(v1, v1)); + DBSPExpression let0 = b.let(b.lit(2, true), x -> b.add(x, x)); + DBSPExpression let1 = b.let(b.lit(2, true), x -> b.add(x, x)); EquivalenceContext context = new EquivalenceContext(); Assert.assertTrue(context.equivalent(let0, let1)); context.leftDeclaration.mustBeEmpty(); @@ -111,104 +93,60 @@ public void testLetEquivalenceContextUnchanged() { @Test public void testCSENested() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i = new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); - DBSPTypeTuple tuple = new DBSPTypeTuple(i, i); - DBSPVariablePath var = tuple.ref().var(); + DBSPTypeTuple tuple = b.tup(b.i32n(), b.i32n()); + DBSPVariablePath var = b.refVar(tuple); DBSPLetStatement stat0 = new DBSPLetStatement("t0", - add(var.deref().field(0), var.deref().field(1))); + b.add(b.field(var, 0), b.field(var, 1))); - DBSPVariablePath t = tuple.ref().var(); + DBSPVariablePath t = b.refVar(tuple); DBSPExpression let = new DBSPLetExpression( t, new DBSPTupleExpression( stat0.getVarReference(), - neg(stat0.getVarReference())).borrow(), - new DBSPTupleExpression(neg(t.deref().field(0)), neg(t.deref().field(0)))); + b.neg(stat0.getVarReference())).borrow(), + b.tuple(b.neg(b.field(t, 0)), b.neg(b.field(t, 0)))); DBSPLetStatement stat1 = new DBSPLetStatement("t1", let); DBSPExpression block = new DBSPBlockExpression( Linq.list(stat0, stat1), stat1.getVarReference()); - DBSPClosureExpression closure = block.closure(var.asParameter()); + DBSPClosureExpression closure = block.closure(var); - DBSPOperator fake = new DBSPConstantOperator( - CalciteEmptyRel.INSTANCE, new DBSPZSetExpression(new DBSPBoolLiteral()), false); - ValueNumbering numbering = new ValueNumbering(compiler); - numbering.setOperatorContext(fake); - numbering.apply(closure); - ExpressionsCSE cse = new ExpressionsCSE(compiler, numbering.canonical); - cse.setOperatorContext(fake); - cse.apply(closure); - IDBSPInnerNode translated = cse.get(closure); - ResolveReferences resolver = new ResolveReferences(compiler, false); // Crash on incorrect translation. - resolver.apply(translated); + ResolveReferences resolver = new ResolveReferences(compiler, false); + resolver.apply(cse(compiler, closure)); } @Test public void testConditionalCSE() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i = new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); - DBSPType b = DBSPTypeBool.create(true); - DBSPTypeTuple tuple = new DBSPTypeTuple(i, i, i); - DBSPVariablePath var = tuple.ref().var(); - DBSPExpression z = new DBSPI32Literal(1); - DBSPExpression cond = new DBSPBinaryExpression( - CalciteObject.EMPTY, b, DBSPOpcode.GTE, var.deref().field(0), z) - .wrapBoolIfNeeded(); - - DBSPExpression un = neg(var.deref().field(1)); - - DBSPExpression if0 = new DBSPIfExpression( - CalciteObject.EMPTY, cond, un, var.deref().field(2)); - DBSPExpression if1 = new DBSPIfExpression( - CalciteObject.EMPTY, cond, var.deref().field(2), un); - - DBSPExpression body = new DBSPTupleExpression(if0, if1); - DBSPClosureExpression closure = body.closure(var.asParameter()); - - ValueNumbering numbering = new ValueNumbering(compiler); - numbering.apply(closure); - ExpressionsCSE cse = new ExpressionsCSE(compiler, numbering.canonical); - DBSPOperator fake = new DBSPConstantOperator( - CalciteEmptyRel.INSTANCE, new DBSPZSetExpression(new DBSPBoolLiteral()), false); - cse.setOperatorContext(fake); - cse.apply(closure); - IDBSPInnerNode translated = cse.get(closure); - - InnerVisitor visitor = new InnerVisitor(compiler) { - int vars = 0; - public void postorder(DBSPLetExpression expression) { - this.vars++; - } - - @Override - public void endVisit() { - // Find 2 common subexpressions - Assert.assertEquals(2, this.vars); - } - }; - visitor.apply(translated); + DBSPClosureExpression closure = b.closure(b.tup(b.i32n(), b.i32n(), b.i32n()), t -> { + DBSPExpression cond = b.binary(DBSPOpcode.GTE, b.field(t, 0), b.lit(1)) + .wrapBoolIfNeeded(); + DBSPExpression un = b.neg(b.field(t, 1)); + return b.tuple( + b.ifThenElse(cond, un, b.field(t, 2)), + b.ifThenElse(cond, b.field(t, 2), un)); + }); + // Find 2 common subexpressions + assertLets(compiler, cse(compiler, closure), 2); } @Test public void testEquiv() { - DBSPLiteral zero0 = new DBSPI32Literal(0); + DBSPExpression zero0 = b.lit(0); DBSPType i32 = zero0.getType(); - DBSPLiteral zero1 = new DBSPI32Literal(0); + DBSPExpression zero1 = b.lit(0); Assert.assertTrue(EquivalenceContext.equiv(zero0, zero1)); - DBSPLiteral one = new DBSPI32Literal(1); + DBSPExpression one = b.lit(1); Assert.assertFalse(EquivalenceContext.equiv(zero0, one)); - DBSPExpression plus0 = new DBSPBinaryExpression( - CalciteObject.EMPTY, zero0.getType(), DBSPOpcode.ADD, zero0, one); - DBSPExpression plus1 = new DBSPBinaryExpression( - CalciteObject.EMPTY, zero0.getType(), DBSPOpcode.ADD, zero1, one); + DBSPExpression plus0 = b.add(zero0, one); + DBSPExpression plus1 = b.add(zero1, one); Assert.assertTrue(EquivalenceContext.equiv(plus0, plus1)); - DBSPExpression plus2 = new DBSPBinaryExpression( - CalciteObject.EMPTY, zero0.getType(), DBSPOpcode.ADD, one, one); + DBSPExpression plus2 = b.add(one, one); Assert.assertFalse(EquivalenceContext.equiv(plus2, plus1)); DBSPVariablePath var0 = new DBSPVariablePath("x", i32); @@ -220,8 +158,7 @@ public void testEquiv() { @SuppressWarnings("SuspiciousNameCombination") @Test public void testLambdas() { - DBSPLiteral zero0 = new DBSPI32Literal(0); - DBSPType i32 = zero0.getType(); + DBSPType i32 = b.i32(); DBSPVariablePath x = new DBSPVariablePath("x", i32); DBSPExpression id0 = x.closure(x); @@ -235,22 +172,19 @@ public void testLambdas() { DBSPVariablePath x2 = new DBSPVariablePath("x", i32); DBSPVariablePath y2 = new DBSPVariablePath("y", i32); - DBSPExpression plus0 = new DBSPBinaryExpression( - CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x2, y2); + DBSPExpression plus0 = b.add(x2, y2); DBSPExpression lambda0 = plus0.closure(x2, y2); DBSPVariablePath x3 = new DBSPVariablePath("x", i32); DBSPVariablePath y3 = new DBSPVariablePath("y", i32); - DBSPExpression plus1 = new DBSPBinaryExpression( - CalciteObject.EMPTY, i32, DBSPOpcode.ADD, y3, x3); + DBSPExpression plus1 = b.add(y3, x3); DBSPExpression lambda1 = plus1.closure(x3, y3); // Compiler doesn't know that ADD is commutative Assert.assertFalse(EquivalenceContext.equiv(lambda0, lambda1)); DBSPVariablePath x4 = new DBSPVariablePath("x", i32); DBSPVariablePath y4 = new DBSPVariablePath("y", i32); - DBSPExpression plus1_1 = new DBSPBinaryExpression( - CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x4, y4); + DBSPExpression plus1_1 = b.add(x4, y4); DBSPExpression lambda2 = plus1_1.closure(x4, y4); Assert.assertTrue(EquivalenceContext.equiv(lambda0, lambda2)); @@ -266,8 +200,8 @@ public void testLambdas() { x.deepCopy().to(DBSPVariablePath.class)); Assert.assertTrue(EquivalenceContext.equiv(blockLambda0, blockLambda1)); - DBSPTypeTuple ii = new DBSPTypeTuple(i32, i32); - DBSPTypeTuple iii = new DBSPTypeTuple(i32, i32, i32); + DBSPTypeTuple ii = b.tup(i32, i32); + DBSPTypeTuple iii = b.tup(i32, i32, i32); DBSPVariablePath x5 = new DBSPVariablePath("x", ii); DBSPVariablePath y5 = new DBSPVariablePath("y", iii); DBSPExpression x0 = x5.field(0).closure(x5); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InliningTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InliningTests.java index c25b58b9472..cc41561a315 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InliningTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InliningTests.java @@ -2,41 +2,33 @@ import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; -import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; -import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; -import org.dbsp.sqlCompiler.ir.type.DBSPType; -import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.dbsp.util.Maybe; import org.junit.Assert; import org.junit.Test; public class InliningTests { + final ExpressionBuilder b = new ExpressionBuilder(); + @Test public void testInlining() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, true); - - DBSPType twice = new DBSPTypeTuple(i32, i32); - DBSPVariablePath x = twice.ref().var(); - // inner = |x: Tup2| Tup3::new(f(x), x.0, x.1) - DBSPClosureExpression inner = new DBSPTupleExpression( - new DBSPApplyExpression("f", i32, x.deref().field(0)), - x.deref().field(0), - x.deref().field(1)).closure(x); - - DBSPVariablePath var = inner.getResultType().ref().var(); - // project = |x: Tup3| Tup2::new(x.0, x.2) - DBSPClosureExpression project = new DBSPTupleExpression( - var.deref().field(0), - var.deref().field(2)).closure(var); + // inner = |x: &Tup2| Tup3::new(f(x.0), x.0, x.1) + DBSPClosureExpression inner = b.closure(b.tup(b.i32n(), b.i32n()), x -> + b.tuple( + b.call(b.i32n(), "f", b.field(x, 0)), + b.field(x, 0), + b.field(x, 1))); + + // project = |v: &Tup3| Tup2::new(v.0, v.2) + DBSPClosureExpression project = b.closure( + inner.getResultType().to(DBSPTypeTuple.class), v -> + b.tuple(b.field(v, 0), b.field(v, 2))); DBSPClosureExpression compose = project.applyAfter(compiler, inner, Maybe.NO); CanonicalForm cf = new CanonicalForm(compiler); @@ -59,23 +51,19 @@ public void testInlining() { @Test public void testLambdaInlining() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, true); - // lambda = |e: &i32| abs(e) - DBSPVariablePath e = i32.ref().var(); - DBSPClosureExpression lambda = new DBSPApplyExpression("abs", i32, e.deref()).closure(e); + DBSPClosureExpression lambda = b.closure(b.i32n(), e -> + b.call(b.i32n(), "abs", e.deref())); - DBSPType tup = new DBSPTypeTuple(i32); - DBSPVariablePath x = tup.ref().var(); // inner = |x: &Tup1| Tup1::new(hof(lambda, x.0)) - DBSPClosureExpression inner = new DBSPTupleExpression( - new DBSPApplyExpression("hof", i32, lambda, x.deref().field(0))).closure(x); + DBSPClosureExpression inner = b.closure(b.tup(b.i32n()), x -> + b.tuple(b.call(b.i32n(), "hof", lambda, b.field(x, 0)))); - DBSPVariablePath v = inner.getResultType().ref().var(); // project = |v: &Tup1| Tup2::new(v.0, v.0): uses the inner value twice, // so inlining copies the lambda into both use sites - DBSPClosureExpression project = new DBSPTupleExpression( - v.deref().field(0), v.deref().field(0)).closure(v); + DBSPClosureExpression project = b.closure( + inner.getResultType().to(DBSPTypeTuple.class), v -> + b.tuple(b.field(v, 0), b.field(v, 0))); DBSPClosureExpression compose = project.applyAfter(compiler, inner, Maybe.YES); // This would crash if the expression is malformed @@ -85,17 +73,15 @@ public void testLambdaInlining() { @Test public void testEnsureTreeWithLambda() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, true); - // lambda = |e: &i32| abs(e) - DBSPVariablePath e = i32.ref().var(); - DBSPClosureExpression lambda = new DBSPApplyExpression("abs", i32, e.deref()).closure(e); + DBSPClosureExpression lambda = b.closure(b.i32n(), e -> + b.call(b.i32n(), "abs", e.deref())); - DBSPType tup = new DBSPTypeTuple(i32); - DBSPVariablePath x = tup.ref().var(); // The same lambda-bearing subtree appears twice: the body is a DAG - DBSPExpression shared = new DBSPApplyExpression("hof", i32, lambda, x.deref().field(0)); - DBSPClosureExpression function = new DBSPTupleExpression(shared, shared).closure(x); + DBSPClosureExpression function = b.closure(b.tup(b.i32n()), x -> { + DBSPExpression shared = b.call(b.i32n(), "hof", lambda, b.field(x, 0)); + return b.tuple(shared, shared); + }); DBSPClosureExpression tree = function.ensureTree(compiler).to(DBSPClosureExpression.class); // The function's own parameters are preserved: analyses key results by them diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InternTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InternTest.java index 38aebc3c60b..762009f8af8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InternTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/InternTest.java @@ -2,55 +2,43 @@ import org.dbsp.sqlCompiler.compiler.CompilerOptions; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; -import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.outer.intern.InternInner; -import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; -import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStringLiteral; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.DBSPTypeInterned; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeFunction; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; import org.junit.Assert; import org.junit.Test; /** Test for the dataflow analysis that performs interning */ public class InternTest { + final ExpressionBuilder b = new ExpressionBuilder(); + @Test public void testIntern() { - DBSPType i = new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); - DBSPTypeString nullableStr = DBSPTypeString.varchar(true); - DBSPTypeString str = DBSPTypeString.varchar(false); - DBSPTypeTuple tuple = new DBSPTypeTuple(i, nullableStr, i, str); - DBSPVariablePath var = tuple.ref().var(); - DBSPExpression add = new DBSPBinaryExpression(CalciteObject.EMPTY, i, DBSPOpcode.ADD, - var.deref().field(0), var.deref().field(1)); - DBSPExpression cmp = new DBSPBinaryExpression(CalciteObject.EMPTY, DBSPTypeBool.create(true), - DBSPOpcode.EQ, var.deref().field(1), var.deref().field(3)); - DBSPExpression concat = new DBSPBinaryExpression(CalciteObject.EMPTY, str, DBSPOpcode.CONCAT, - var.deref().field(1), new DBSPStringLiteral(" hello")); - DBSPApplyExpression len = new DBSPApplyExpression("len", i, concat); - DBSPExpression cond = new DBSPIfExpression(CalciteObject.EMPTY, - cmp.wrapBoolIfNeeded(), add, len); - DBSPExpression result = new DBSPTupleExpression(cond, var.deref().field(3)); - DBSPClosureExpression closure = result.closure(var); + DBSPTypeTuple tuple = b.tup(b.i32n(), b.strn(), b.i32n(), b.str()); + DBSPClosureExpression closure = b.closure(tuple, t -> { + DBSPExpression add = b.add(b.field(t, 0), b.field(t, 1)); + DBSPExpression cmp = b.binary(DBSPOpcode.EQ, b.field(t, 1), b.field(t, 3)); + DBSPExpression concat = b.binary(b.str(), DBSPOpcode.CONCAT, + b.field(t, 1), b.lit(" hello")); + DBSPExpression len = b.call(b.i32n(), "len", concat); + DBSPExpression cond = b.ifThenElse(cmp.wrapBoolIfNeeded(), add, len); + return b.tuple(cond, b.field(t, 3)); + }); DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPType parameterType = new DBSPTypeTuple(i, DBSPTypeInterned.INSTANCE, i, DBSPTypeInterned.INSTANCE); + DBSPType parameterType = b.tup( + b.i32n(), DBSPTypeInterned.INSTANCE, b.i32n(), DBSPTypeInterned.INSTANCE); InternInner ii = new InternInner(compiler, true, false, parameterType.ref()); DBSPExpression converted = ii.apply(closure).to(DBSPExpression.class); DBSPType convertedType = converted.getType(); DBSPType expectedType = new DBSPTypeFunction( - new DBSPTypeTuple(cond.getType(), DBSPTypeInterned.INSTANCE), parameterType.ref()); + b.tup(b.i32n(), DBSPTypeInterned.INSTANCE), parameterType.ref()); Assert.assertTrue(convertedType.sameType(expectedType)); } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java index 1bbadef2c2e..ad3edeaee0d 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/TestSimplifyConditionals.java @@ -1,40 +1,27 @@ package org.dbsp.sqlCompiler.compiler.ir; import org.dbsp.sqlCompiler.compiler.DBSPCompiler; -import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify; import org.dbsp.sqlCompiler.compiler.visitors.inner.SimplifyConditionals; -import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; -import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; -import org.dbsp.sqlCompiler.ir.expression.literal.DBSPIntLiteral; -import org.dbsp.sqlCompiler.ir.type.DBSPType; -import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.junit.Assert; import org.junit.Test; /** Unit tests for {@link SimplifyConditionals} */ public class TestSimplifyConditionals extends BaseSQLTests { + final ExpressionBuilder b = new ExpressionBuilder(); + @Test public void testVariable() { DBSPCompiler compiler = this.testCompiler(); - DBSPType b = DBSPTypeBool.INSTANCE; - DBSPVariablePath x = b.var(); - // inner = |x: boolean| if (x) { x } else { !x } - DBSPClosureExpression clo = new DBSPIfExpression( - CalciteObject.EMPTY, - x.deepCopy(), - x.deepCopy(), - x.deepCopy().not()).closure(x); + // clo = |x: boolean| if (x) { x } else { !x } + var clo = b.lambda(b.bool(), x -> + b.ifThenElse(x.deepCopy(), x.deepCopy(), x.deepCopy().not())); SimplifyConditionals sc = new SimplifyConditionals(compiler); var result = sc.apply(clo); @@ -60,15 +47,6 @@ public void testVariable() { @Test public void testComplexComparison() { DBSPCompiler compiler = this.testCompiler(); - DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, false); - DBSPType b = DBSPTypeBool.INSTANCE; - - DBSPVariablePath x = i32.var(); - DBSPIntLiteral two = new DBSPI32Literal(CalciteObject.EMPTY, i32, 2); - DBSPIntLiteral one = new DBSPI32Literal(CalciteObject.EMPTY, i32, 1); - DBSPIntLiteral zero = new DBSPI32Literal(CalciteObject.EMPTY, i32, 0); - DBSPExpression xPlusOne = new DBSPBinaryExpression(CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x, one); - DBSPExpression lZ = new DBSPBinaryExpression(CalciteObject.EMPTY, b, DBSPOpcode.LT, xPlusOne, zero); // |x: i32| { // if ((x + 1) < 0) { // if ((x + 1) < 0) { 0 } else { 1 } @@ -76,16 +54,13 @@ public void testComplexComparison() { // 2 // } // } - var innerIf = new DBSPIfExpression( - CalciteObject.EMPTY, - lZ.deepCopy(), - zero.deepCopy(), - one.deepCopy()); - var clo = new DBSPIfExpression( - CalciteObject.EMPTY, - lZ.deepCopy(), - innerIf, - two).closure(x); + var clo = b.lambda(b.i32(), x -> { + DBSPExpression lZ = b.binary(DBSPOpcode.LT, b.add(x, b.lit(1)), b.lit(0)); + return b.ifThenElse( + lZ.deepCopy(), + b.ifThenElse(lZ.deepCopy(), b.lit(0), b.lit(1)), + b.lit(2)); + }); CanonicalForm cf = new CanonicalForm(compiler); SimplifyConditionals sc = new SimplifyConditionals(compiler); @@ -102,7 +77,7 @@ public void testComplexComparison() { } else { 2 }))""", result.toString()); - + Simplify simplify = new Simplify(compiler); result = simplify.apply(result); result = cf.apply(result); @@ -118,16 +93,8 @@ public void testComplexComparison() { @Test public void testAliasedVariable() { DBSPCompiler compiler = this.testCompiler(); - DBSPType i32 = DBSPTypeInteger.getType(CalciteObject.EMPTY, DBSPTypeCode.INT32, false); - DBSPType b = DBSPTypeBool.INSTANCE; - - // The compiler never reuses variable names, but this is supposed to work too. - DBSPVariablePath x = i32.var(); - DBSPIntLiteral two = new DBSPI32Literal(CalciteObject.EMPTY, i32, 2); - DBSPIntLiteral one = new DBSPI32Literal(CalciteObject.EMPTY, i32, 1); - DBSPIntLiteral zero = new DBSPI32Literal(CalciteObject.EMPTY, i32, 0); - DBSPExpression xPlusOne = new DBSPBinaryExpression(CalciteObject.EMPTY, i32, DBSPOpcode.ADD, x, one); - DBSPExpression lZ = new DBSPBinaryExpression(CalciteObject.EMPTY, b, DBSPOpcode.LT, xPlusOne, zero); + // The compiler never reuses variable names, but this is supposed to work + // too; the let deliberately rebinds the lambda's own parameter node. // inner = |x: i32| { // if ((x + 1) < 0) { // let x = x + 1; @@ -135,17 +102,13 @@ public void testAliasedVariable() { // } else { // 2 // } - var innerIf = new DBSPIfExpression( - CalciteObject.EMPTY, - lZ.deepCopy(), - zero.deepCopy(), - one.deepCopy()); - var let = new DBSPLetExpression(x, xPlusOne.deepCopy(), innerIf); - var clo = new DBSPIfExpression( - CalciteObject.EMPTY, - lZ.deepCopy(), - let, - two).closure(x); + var clo = b.lambda(b.i32(), x -> { + DBSPExpression xPlusOne = b.add(x, b.lit(1)); + DBSPExpression lZ = b.binary(DBSPOpcode.LT, xPlusOne, b.lit(0)); + var innerIf = b.ifThenElse(lZ.deepCopy(), b.lit(0), b.lit(1)); + var let = new DBSPLetExpression(x, xPlusOne.deepCopy(), innerIf); + return b.ifThenElse(lZ.deepCopy(), let, b.lit(2)); + }); CanonicalForm cf = new CanonicalForm(compiler); var initial = cf.apply(clo); Assert.assertEquals(""" diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/UnusedFieldsTest.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/UnusedFieldsTest.java index d6ea03d99e9..6ac7c8a951e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/UnusedFieldsTest.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/ir/UnusedFieldsTest.java @@ -4,6 +4,7 @@ import org.dbsp.sqlCompiler.compiler.DBSPCompiler; import org.dbsp.sqlCompiler.compiler.backend.rust.ToRustInnerVisitor; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FieldUseMap; import org.dbsp.sqlCompiler.compiler.visitors.unusedFields.FindUsedFields; @@ -12,12 +13,10 @@ import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; -import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; import org.dbsp.sqlCompiler.ir.expression.DBSPZSetExpression; import org.dbsp.sqlCompiler.ir.type.DBSPType; import org.dbsp.sqlCompiler.ir.type.DBSPTypeCode; import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; -import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; import org.dbsp.sqlCompiler.ir.type.user.DBSPTypeArray; @@ -28,6 +27,8 @@ import java.util.Objects; public class UnusedFieldsTest { + final ExpressionBuilder b = new ExpressionBuilder(); + @Test public void testZSetString() { DBSPExpression none = new DBSPTypeArray(DBSPTypeString.varchar(false), true).none(); @@ -52,23 +53,13 @@ public void testZSetString() { @Test public void testReduce() { DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); - DBSPTypeTuple tuple = new DBSPTypeTuple( - new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true), - DBSPTypeBool.create(false), - DBSPTypeBool.create(true), - new DBSPTypeTuple( - DBSPTypeString.varchar(true), - DBSPTypeString.varchar(false))); - - DBSPVariablePath var0 = tuple.ref().var(); - DBSPExpression body0 = new DBSPTupleExpression( - var0.deref().field(1), - var0.deref().field(3).field(0)); - DBSPClosureExpression closure0 = body0.closure(var0.asParameter()); - - DBSPVariablePath var1 = tuple.ref().var(); - DBSPExpression body1 = new DBSPTupleExpression(var1.deref().field(0)); - DBSPClosureExpression closure1 = body1.closure(var1.asParameter()); + DBSPTypeTuple tuple = b.tup( + b.i32n(), b.bool(), b.booln(), b.tup(b.strn(), b.str())); + + DBSPClosureExpression closure0 = b.closure(tuple, t -> + b.tuple(b.field(t, 1), b.field(t, 3).field(0))); + DBSPClosureExpression closure1 = b.closure(tuple, t -> + b.tuple(b.field(t, 0))); FindUsedFields fuf = new FindUsedFields(compiler); ParameterFieldUse fum = fuf.findUsedFields(closure0); @@ -86,19 +77,13 @@ public void testReduce() { @Test public void unusedFieldsTest() { - DBSPTypeTuple tuple = new DBSPTypeTuple( - new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true), - DBSPTypeBool.create(false), - DBSPTypeBool.create(true), - new DBSPTypeTuple( - DBSPTypeString.varchar(true), - DBSPTypeString.varchar(false))); - DBSPVariablePath var = tuple.ref().var(); - DBSPExpression body = new DBSPTupleExpression( - var.deref().field(0), - var.deref().field(2), - var.deref().field(3).field(0).applyClone()); - DBSPClosureExpression closure = body.closure(var.asParameter()); + DBSPTypeTuple tuple = b.tup( + b.i32n(), b.bool(), b.booln(), b.tup(b.strn(), b.str())); + DBSPClosureExpression closure = b.closure(tuple, t -> + b.tuple( + b.field(t, 0), + b.field(t, 2), + b.field(t, 3).field(0).applyClone())); DBSPCompiler compiler = new DBSPCompiler(new CompilerOptions()); CanonicalForm cf = new CanonicalForm(compiler); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/ExpressionBuilder.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/ExpressionBuilder.java new file mode 100644 index 00000000000..dfbd811922e --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/tools/ExpressionBuilder.java @@ -0,0 +1,171 @@ +package org.dbsp.sqlCompiler.compiler.sql.tools; + +import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBinaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPIfExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPOpcode; +import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPUnaryExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPI32Literal; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStringLiteral; +import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; +import org.dbsp.sqlCompiler.ir.statement.DBSPStatement; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeBool; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeInteger; +import org.dbsp.sqlCompiler.ir.type.primitive.DBSPTypeString; + +import java.util.List; +import java.util.function.Function; + +/** Builds small inner-IR expressions for unit tests. + * Binding constructs (let, block, lambda, closure) take a Java function + * that receives the bound variable, so the binder and its uses share the + * correct node identity by construction. */ +// Example: +// DBSPClosureExpression f = b.closure(b.tup(b.i32(), b.i32()), t -> +// b.let(b.call("f", b.field(t, 0)), +// x -> b.call("g", x))); +// builds the closure that compiles to the Rust code +// move |t_0: &Tup2| -> i32 { +// let t_1 = f((*t_0).0); +// g(t_1) +// } +public class ExpressionBuilder { + public DBSPType i32() { + return new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, false); + } + + /** Nullable INT */ + public DBSPType i32n() { + return new DBSPTypeInteger(CalciteObject.EMPTY, 32, true, true); + } + + /** VARCHAR */ + public DBSPType str() { + return DBSPTypeString.varchar(false); + } + + /** Nullable VARCHAR */ + public DBSPType strn() { + return DBSPTypeString.varchar(true); + } + + public DBSPType bool() { + return new DBSPTypeBool(CalciteObject.EMPTY, false); + } + + /** Nullable BOOLEAN */ + public DBSPType booln() { + return new DBSPTypeBool(CalciteObject.EMPTY, true); + } + + public DBSPTypeTuple tup(DBSPType... fields) { + return new DBSPTypeTuple(fields); + } + + public DBSPExpression lit(int value) { + return new DBSPI32Literal(value); + } + + /** An INT literal, nullable if requested */ + public DBSPExpression lit(int value, boolean nullable) { + return new DBSPI32Literal(value, nullable); + } + + public DBSPExpression lit(String value) { + return new DBSPStringLiteral(value); + } + + /** A variable of the reference type; becomes a parameter when a closure is built over it */ + public DBSPVariablePath refVar(DBSPType type) { + return type.ref().var(); + } + + /** Field of a row parameter */ + public DBSPExpression field(DBSPVariablePath row, int index) { + return row.deref().field(index); + } + + /** A call to a UDF returning INT; any external call is expensive */ + public DBSPExpression call(String function, DBSPExpression... args) { + return this.call(this.i32(), function, args); + } + + public DBSPExpression call(DBSPType returnType, String function, DBSPExpression... args) { + return new DBSPApplyExpression(function, returnType, args); + } + + public DBSPExpression add(DBSPExpression left, DBSPExpression right) { + return this.binary(DBSPOpcode.ADD, left, right); + } + + public DBSPExpression binary(DBSPOpcode opcode, DBSPExpression left, DBSPExpression right) { + // Comparing a nullable value produces a nullable Boolean + DBSPType type = opcode.isComparison() + ? new DBSPTypeBool(CalciteObject.EMPTY, + left.getType().mayBeNull || right.getType().mayBeNull) + : left.getType(); + return this.binary(type, opcode, left, right); + } + + /** A binary expression with an explicit result type */ + public DBSPExpression binary(DBSPType type, DBSPOpcode opcode, + DBSPExpression left, DBSPExpression right) { + return new DBSPBinaryExpression(CalciteObject.EMPTY, type, opcode, left, right); + } + + public DBSPExpression unary(DBSPOpcode opcode, DBSPExpression source) { + return new DBSPUnaryExpression(CalciteObject.EMPTY, source.getType(), opcode, source); + } + + public DBSPExpression neg(DBSPExpression source) { + return this.unary(DBSPOpcode.NEG, source); + } + + public DBSPExpression ifThenElse(DBSPExpression condition, + DBSPExpression positive, DBSPExpression negative) { + return new DBSPIfExpression(CalciteObject.EMPTY, condition, positive, negative); + } + + public DBSPExpression tuple(DBSPExpression... fields) { + return new DBSPTupleExpression(fields); + } + + /** let var = initializer; consumer(var) */ + public DBSPExpression let(DBSPExpression initializer, + Function consumer) { + DBSPVariablePath var = initializer.getType().var(); + return new DBSPLetExpression(var, initializer, consumer.apply(var)); + } + + /** { let var = initializer; last(var) } */ + public DBSPExpression block(DBSPExpression initializer, + Function last) { + DBSPVariablePath var = initializer.getType().var(); + DBSPStatement statement = new DBSPLetStatement(var.variable, initializer); + return new DBSPBlockExpression(List.of(statement), last.apply(var)); + } + + /** |var| body(var), a nested lambda over a value */ + public DBSPClosureExpression lambda(DBSPType paramType, + Function body) { + DBSPVariablePath var = paramType.var(); + return body.apply(var).closure(var); + } + + /** A closure over a parameter passed by reference, + * the shape of a map function when the type is a row type */ + public DBSPClosureExpression closure(DBSPType paramType, + Function body) { + DBSPVariablePath var = this.refVar(paramType); + return body.apply(var).closure(var); + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferencesTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferencesTests.java new file mode 100644 index 00000000000..f0b69c2e7d3 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/inner/ResolveReferencesTests.java @@ -0,0 +1,133 @@ +package org.dbsp.sqlCompiler.compiler.visitors.inner; + +import org.dbsp.sqlCompiler.compiler.errors.InternalCompilerError; +import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; +import org.dbsp.sqlCompiler.ir.expression.DBSPBlockExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPLetExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.statement.DBSPLetStatement; +import org.junit.Assert; +import org.junit.Test; + +/** Tests for {@link ResolveReferences}. */ +public class ResolveReferencesTests extends BaseSQLTests { + final ExpressionBuilder b = new ExpressionBuilder(); + + ReferenceMap resolve(DBSPExpression expression, boolean allowFree) { + ResolveReferences resolver = new ResolveReferences(this.testCompiler(), allowFree); + resolver.apply(expression); + return resolver.reference; + } + + /** Uses of the closure parameter resolve to the parameter */ + @Test + public void testParameterResolves() { + DBSPVariablePath t = b.refVar(b.tup(b.i32(), b.i32())); + DBSPClosureExpression closure = b.add(b.field(t, 0), b.field(t, 1)).closure(t); + ReferenceMap references = this.resolve(closure, false); + Assert.assertSame(closure.parameters[0], references.get(t)); + } + + /** Let expression */ + @Test + public void testLetResolves() { + DBSPVariablePath t = b.refVar(b.tup(b.i32(), b.i32())); + DBSPVariablePath[] bound = new DBSPVariablePath[1]; + DBSPExpression let = b.let(b.call("f", b.field(t, 0)), + x -> { + bound[0] = x; + return b.call("g", x); + }); + ReferenceMap references = this.resolve(let.closure(t), false); + Assert.assertSame(let, references.get(bound[0])); + } + + @Test + public void testLetBinderDistinctNode() { + DBSPVariablePath t = b.refVar(b.tup(b.i32(), b.i32())); + DBSPVariablePath binder = b.i32().var(); + DBSPVariablePath use = new DBSPVariablePath(binder.variable, b.i32()); + DBSPExpression let = new DBSPLetExpression( + binder, b.call("f", b.field(t, 0)), b.call("g", use)); + DBSPClosureExpression closure = b.tuple(let).closure(t); + + ReferenceMap references = this.resolve(closure, false); + Assert.assertSame(let, references.get(binder)); + Assert.assertSame(let, references.get(use)); + + // ValueNumbering visits the binder node and enforces that it resolves + new ValueNumbering(this.testCompiler()).apply(closure); + } + + /** A let initializer resolves in the scope outside the let: + * in let x = 1; let x = x + 1; x + * the second initializer sees the first x, the consumer the second. */ + @Test + public void testShadowing() { + DBSPVariablePath outerBinder = b.i32().var(); + String name = outerBinder.variable; + DBSPVariablePath innerBinder = new DBSPVariablePath(name, b.i32()); + DBSPVariablePath initializerUse = new DBSPVariablePath(name, b.i32()); + DBSPVariablePath consumerUse = new DBSPVariablePath(name, b.i32()); + DBSPLetExpression inner = new DBSPLetExpression( + innerBinder, b.add(initializerUse, b.lit(1)), consumerUse); + DBSPLetExpression outer = new DBSPLetExpression(outerBinder, b.lit(1), inner); + + ReferenceMap references = this.resolve(outer, false); + Assert.assertSame(outer, references.get(initializerUse)); + Assert.assertSame(inner, references.get(consumerUse)); + Assert.assertSame(outer, references.get(outerBinder)); + Assert.assertSame(inner, references.get(innerBinder)); + } + + /** A lambda parameter shadows an outer variable with the same name */ + @Test + public void testLambdaShadowing() { + DBSPVariablePath outerParam = b.i32().var(); + String name = outerParam.variable; + DBSPVariablePath outerUse = new DBSPVariablePath(name, b.i32()); + DBSPVariablePath innerParam = new DBSPVariablePath(name, b.i32()); + DBSPVariablePath innerUse = new DBSPVariablePath(name, b.i32()); + DBSPClosureExpression inner = b.call("h", innerUse).closure(innerParam); + DBSPClosureExpression outer = b.call("m", outerUse, inner).closure(outerParam); + + ReferenceMap references = this.resolve(outer, false); + Assert.assertSame(outer.parameters[0], references.get(outerUse)); + Assert.assertSame(inner.parameters[0], references.get(innerUse)); + } + + /** A variable declared by a let statement resolves to the statement */ + @Test + public void testBlockStatement() { + DBSPVariablePath t = b.refVar(b.tup(b.i32(), b.i32())); + DBSPVariablePath[] bound = new DBSPVariablePath[1]; + DBSPExpression block = b.block(b.call("f", b.field(t, 0)), + s -> { + bound[0] = s; + return b.add(s, b.lit(1)); + }); + ReferenceMap references = this.resolve(block.closure(t), false); + DBSPLetStatement statement = block.to(DBSPBlockExpression.class) + .contents.get(0).to(DBSPLetStatement.class); + Assert.assertSame(statement, references.get(bound[0])); + } + + /** A free variable is tolerated only when allowed */ + @Test + public void testFreeVariable() { + DBSPVariablePath free = b.i32().var(); + DBSPExpression expression = b.add(free, b.lit(1)); + + // The only test that needs the resolver itself, for the flag + ResolveReferences resolver = new ResolveReferences(this.testCompiler(), true); + resolver.apply(expression); + Assert.assertTrue(resolver.freeVariablesFound); + Assert.assertNull(resolver.reference.get(free)); + + Assert.assertThrows(InternalCompilerError.class, + () -> this.resolve(expression, false)); + } +} From db9d239f26ae90bc0bf33b0a4a92b3937a2bce98 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 7 Aug 2026 14:25:13 -0700 Subject: [PATCH 2/3] [SQL] Rename EndToEndTests to BasicQueriesTests Signed-off-by: Mihai Budiu --- .../dbsp/sqlCompiler/compiler/sql/OtherTests.java | 6 +++--- .../{EndToEndTests.java => BasicQueriesTests.java} | 13 ++++++------- .../compiler/sql/simple/MultiViewTests.java | 12 ++++++------ .../compiler/sql/simple/NaiveIncrementalTests.java | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) rename sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/{EndToEndTests.java => BasicQueriesTests.java} (99%) diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java index 5a5474db250..0d6fc0726b8 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/OtherTests.java @@ -48,7 +48,7 @@ import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlToRelCompiler; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.FunctionDocumentation; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; -import org.dbsp.sqlCompiler.compiler.sql.simple.EndToEndTests; +import org.dbsp.sqlCompiler.compiler.sql.simple.BasicQueriesTests; import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; import org.dbsp.sqlCompiler.compiler.sql.tools.Change; import org.dbsp.sqlCompiler.compiler.sql.tools.CompilerCircuit; @@ -195,7 +195,7 @@ public void loggingParameter() throws IOException, InterruptedException, SQLExce @Test public void toCsvTest() { DBSPCompiler compiler = testCompiler(); - DBSPZSetExpression s = new DBSPZSetExpression(EndToEndTests.E0, EndToEndTests.E1); + DBSPZSetExpression s = new DBSPZSetExpression(BasicQueriesTests.E0, BasicQueriesTests.E1); StringBuilder builder = new StringBuilder(); ToCsvVisitor visitor = new ToCsvVisitor(compiler, builder, () -> ""); visitor.apply(s); @@ -210,7 +210,7 @@ public void toCsvTest() { @Test public void rustCsvTest() throws IOException, InterruptedException { DBSPCompiler compiler = testCompiler(); - DBSPZSetExpression data = new DBSPZSetExpression(EndToEndTests.E0, EndToEndTests.E1); + DBSPZSetExpression data = new DBSPZSetExpression(BasicQueriesTests.E0, BasicQueriesTests.E1); File file = File.createTempFile("test", ".csv", new File(BaseSQLTests.RUST_DIRECTORY)); file.deleteOnExit(); ToCsvVisitor.toCsv(compiler, file, data); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/BasicQueriesTests.java similarity index 99% rename from sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java rename to sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/BasicQueriesTests.java index a83d7d3df5a..86e24af4f48 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/EndToEndTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/BasicQueriesTests.java @@ -61,12 +61,11 @@ import java.math.BigDecimal; /** - * Test end-to-end by compiling some DDL statements and view - * queries by compiling them to rust and executing them - * by inserting data in the input tables and reading data - * from the declared views. */ -public class EndToEndTests extends BaseSQLTests { - public static final String E2E_TABLE = """ + * Basic queries over a single table. + * Each view is compiled to Rust and executed by inserting + * data in the input table and reading data from the view. */ +public class BasicQueriesTests extends BaseSQLTests { + public static final String BASIC_TABLE = """ CREATE TABLE T ( COL1 INT NOT NULL , COL2 DOUBLE PRECISION NOT NULL @@ -109,7 +108,7 @@ CREATE TABLE T ( public DBSPCompiler compileQuery(String query) { DBSPCompiler compiler = this.testCompiler(); - compiler.submitStatementForCompilation(E2E_TABLE); + compiler.submitStatementForCompilation(BASIC_TABLE); compiler.submitStatementForCompilation(query); return compiler; } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MultiViewTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MultiViewTests.java index 5ba27d1511a..e4ac3cb4a2c 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MultiViewTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/MultiViewTests.java @@ -45,12 +45,12 @@ public void twoViewTest() { String query2 = "CREATE VIEW V2 as SELECT T.COL2 FROM T"; DBSPCompiler compiler = testCompiler(); - compiler.submitStatementForCompilation(EndToEndTests.E2E_TABLE); + compiler.submitStatementForCompilation(BasicQueriesTests.BASIC_TABLE); compiler.submitStatementForCompilation(query1); compiler.submitStatementForCompilation(query2); CompilerCircuitStream ccs = this.getCCS(compiler); - Change inputChange = EndToEndTests.INPUT; + Change inputChange = BasicQueriesTests.INPUT; Change outputChange = new Change( new TableData("V1", new DBSPZSetExpression( new DBSPTupleExpression(new DBSPBoolLiteral(true)), @@ -68,13 +68,13 @@ public void nestedViewTest() { String query2 = "CREATE VIEW V2 as SELECT * FROM V1"; DBSPCompiler compiler = testCompiler(); - compiler.submitStatementForCompilation(EndToEndTests.E2E_TABLE); + compiler.submitStatementForCompilation(BasicQueriesTests.BASIC_TABLE); compiler.submitStatementForCompilation(query1); compiler.submitStatementForCompilation(query2); CompilerCircuitStream ccs = this.getCCS(compiler); InputOutputChange change = new InputOutputChange( - EndToEndTests.INPUT, + BasicQueriesTests.INPUT, new Change( new TableData("V1", new DBSPZSetExpression( new DBSPTupleExpression(new DBSPBoolLiteral(true)), @@ -92,13 +92,13 @@ public void multiViewTest() { String query2 = "CREATE VIEW V2 as SELECT DISTINCT COL1 FROM (SELECT * FROM V1 JOIN T ON V1.COL3 = T.COL3)"; DBSPCompiler compiler = testCompiler(); - compiler.submitStatementForCompilation(EndToEndTests.E2E_TABLE); + compiler.submitStatementForCompilation(BasicQueriesTests.BASIC_TABLE); compiler.submitStatementForCompilation(query1); compiler.submitStatementForCompilation(query2); CompilerCircuitStream ccs = this.getCCS(compiler); InputOutputChange change = new InputOutputChange( - EndToEndTests.INPUT, + BasicQueriesTests.INPUT, new Change(new TableData("V1", new DBSPZSetExpression( new DBSPTupleExpression(new DBSPBoolLiteral(true)), new DBSPTupleExpression(new DBSPBoolLiteral(false)))), diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java index bf451243d9d..2d54448a81e 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/NaiveIncrementalTests.java @@ -31,7 +31,7 @@ // Runs the EndToEnd tests but on an input stream with 3 elements each and // using an incremental non-optimized circuit. -public class NaiveIncrementalTests extends EndToEndTests { +public class NaiveIncrementalTests extends BasicQueriesTests { @Override public CompilerOptions testOptions() { CompilerOptions options = super.testOptions(); From ad5b51aaf5adeda0d914b420f798973264f852f0 Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Fri, 7 Aug 2026 16:55:05 -0700 Subject: [PATCH 3/3] [SQL] Optimization to share expensive expressions across sibling map operators Signed-off-by: Mihai Budiu --- .../src/compiler/sql_compiler.rs | 8 +- crates/pipeline-manager/src/compiler/test.rs | 5 +- .../compiler/visitors/inner/Expensive.java | 2 - .../visitors/outer/CircuitOptimizer.java | 2 + .../visitors/outer/ConstantViews.java | 44 +++ .../visitors/outer/FuseExpensiveMaps.java | 356 ++++++++++++++++++ .../ir/expression/DBSPClosureExpression.java | 10 + .../src/main/java/org/dbsp/util/Logger.java | 3 +- .../CollectExpensiveExpressionsTests.java | 217 +++++++++++ .../outer/FuseExpensiveMapsTests.java | 169 +++++++++ 10 files changed, 808 insertions(+), 8 deletions(-) create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ConstantViews.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMaps.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CollectExpensiveExpressionsTests.java create mode 100644 sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMapsTests.java diff --git a/crates/pipeline-manager/src/compiler/sql_compiler.rs b/crates/pipeline-manager/src/compiler/sql_compiler.rs index 7e98453e565..0026d229856 100644 --- a/crates/pipeline-manager/src/compiler/sql_compiler.rs +++ b/crates/pipeline-manager/src/compiler/sql_compiler.rs @@ -1165,7 +1165,7 @@ mod test { .create_pipeline(tenant_id, "p1", "v0", program_code) .await; test.sql_compiler_tick().await; - test.check_outcome_sql_compiled(tenant_id, pipeline_id, program_code) + test.check_outcome_sql_compiled(tenant_id, pipeline_id, program_code, true) .await; test.delete_pipeline(tenant_id, pipeline_id, "p1").await; test.sql_compiler_tick().await; @@ -1218,7 +1218,7 @@ mod test { .await; test.sql_compiler_tick().await; let pipeline_descr = test - .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code) + .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code, false) .await; // Check the types of the table and view @@ -1406,7 +1406,7 @@ mod test { .await; test.sql_compiler_tick().await; let pipeline_descr = test - .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code) + .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code, false) .await; // Check materialized outcome @@ -1471,7 +1471,7 @@ mod test { // Check result let pipeline_descr = test - .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code) + .check_outcome_sql_compiled(tenant_id, pipeline_id, program_code, false) .await; let input_connectors = validate_program_info(&pipeline_descr.program_info.clone().unwrap()) .unwrap() diff --git a/crates/pipeline-manager/src/compiler/test.rs b/crates/pipeline-manager/src/compiler/test.rs index 0815641ebdc..2735fa22160 100644 --- a/crates/pipeline-manager/src/compiler/test.rs +++ b/crates/pipeline-manager/src/compiler/test.rs @@ -218,6 +218,7 @@ impl CompilerTest { tenant_id: TenantId, pipeline_id: PipelineId, program_code: &str, + warnings: bool, ) -> ExtendedPipelineDescr { // Retrieve pipeline descriptor let pipeline_descr = self.get_pipeline(tenant_id, pipeline_id).await; @@ -271,7 +272,9 @@ impl CompilerTest { ); assert_eq!(content_program_sql, program_code); assert_ne!(content_schema_json, ""); - assert_eq!(content_stderr_log, ""); + if (!warnings) { + assert_eq!(content_stderr_log, ""); + } assert_eq!(content_stdout_log, ""); // Return the pipeline descriptor diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Expensive.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Expensive.java index 25626cf22cc..70f9cb63fcf 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Expensive.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/inner/Expensive.java @@ -72,7 +72,6 @@ public VisitDecision preorder(DBSPApplyMethodExpression unused) { @Override public VisitDecision preorder(DBSPBinaryExpression expression) { - // Lowered to a runtime function call if (expression.opcode == DBSPOpcode.VARIANT_INDEX) { this.expensive = true; return VisitDecision.STOP; @@ -82,7 +81,6 @@ public VisitDecision preorder(DBSPBinaryExpression expression) { @Override public VisitDecision preorder(DBSPCastExpression expression) { - // VARIANT casts are lowered to runtime function calls if (expression.getType().is(DBSPTypeVariant.class) || expression.source.getType().is(DBSPTypeVariant.class)) { this.expensive = true; diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java index afc57bc7613..76ba437b8f5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CircuitOptimizer.java @@ -96,6 +96,7 @@ void createOptimizer() { AnalyzedSet operatorsAnalyzed = new AnalyzedSet<>(); this.add(new OptimizeWithGraph(compiler, g -> new OptimizeProjections(compiler, true, g, operatorsAnalyzed), 1)); + this.add(new FuseExpensiveMaps(compiler)); this.add(new RemoveViewOperators(compiler, false)); this.add(new UnusedFields(compiler)); this.add(new Intern(compiler)); @@ -173,6 +174,7 @@ void createOptimizer() { this.add(new OptimizeWithGraph(compiler, g -> new StrayGC(compiler, g))); // The canonical form is needed if we want the Merkle hashes to be "stable". this.add(new CanonicalForm(compiler).getCircuitRewriter(false)); + this.add(new ConstantViews(compiler)); this.add(new StaticDeclarations(compiler, new ImplementStatics(compiler, !compiler.options.ioOptions.multiCrates()))); // From now on we cannot really change the graph anymore. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ConstantViews.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ConstantViews.java new file mode 100644 index 00000000000..e988c58f4fa --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/ConstantViews.java @@ -0,0 +1,44 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.circuit.OutputPort; +import org.dbsp.sqlCompiler.circuit.operator.DBSPConstantOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPDifferentiateOperator; +import org.dbsp.sqlCompiler.circuit.operator.DBSPSinkOperator; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; + +import java.util.HashSet; +import java.util.Set; + +/** Give a warning when a view is fed from a constant operator + * (via potentially a differentiator) */ +public class ConstantViews extends CircuitVisitor { + final Set constant = new HashSet<>(); + + public ConstantViews(DBSPCompiler compiler) { + super(compiler); + } + + @Override + public void postorder(DBSPConstantOperator operator) { + this.constant.add(operator.outputPort()); + } + + @Override + public void postorder(DBSPDifferentiateOperator operator) { + if (this.constant.contains(operator.input())) { + this.constant.add(operator.outputPort()); + } + } + + @Override + public void postorder(DBSPSinkOperator operator) { + if (operator.viewName.equals(DBSPCompiler.ERROR_VIEW_NAME)) + return; + if (this.constant.contains(operator.input())) { + this.compiler.reportWarning( + operator.getSourcePosition(), + "View is constant", + "View " + operator.viewName.singleQuote() + " does not depend on any input data"); + } + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMaps.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMaps.java new file mode 100644 index 00000000000..11627077950 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMaps.java @@ -0,0 +1,356 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.circuit.OutputPort; +import org.dbsp.sqlCompiler.circuit.annotation.IsProjection; +import org.dbsp.sqlCompiler.circuit.operator.DBSPMapOperator; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; +import org.dbsp.sqlCompiler.compiler.visitors.inner.CanonicalForm; +import org.dbsp.sqlCompiler.compiler.visitors.inner.EquivalenceContext; +import org.dbsp.sqlCompiler.compiler.visitors.inner.Expensive; +import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; +import org.dbsp.sqlCompiler.compiler.visitors.inner.ReferenceMap; +import org.dbsp.sqlCompiler.compiler.visitors.inner.ResolveReferences; +import org.dbsp.sqlCompiler.compiler.visitors.inner.Simplify; +import org.dbsp.sqlCompiler.compiler.visitors.outer.temporal.ContainsNow; +import org.dbsp.sqlCompiler.ir.DBSPParameter; +import org.dbsp.sqlCompiler.ir.IDBSPDeclaration; +import org.dbsp.sqlCompiler.ir.IDBSPInnerNode; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPForExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; +import org.dbsp.util.Utilities; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** Fuses one or more sibling {@link DBSPMapOperator}s that share + * expensive common subexpressions into a single map followed by one projection + * per original map. + * Maps whose functions call now() are left alone. */ +public class FuseExpensiveMaps extends Passes { + public FuseExpensiveMaps(DBSPCompiler compiler) { + super("FuseExpensiveMaps", compiler); + FindFusableMaps find = new FindFusableMaps(compiler); + this.add(find); + this.add(new Fuse(compiler, find)); + } + + /** A map that may be fused with siblings. + * @param operator The original map operator. + * @param function A single-parameter closure with a tuple body. + * @param minimal Minimal expensive subexpressions of the function body. */ + record MapInfo(DBSPMapOperator operator, DBSPClosureExpression function, + List minimal) { + DBSPParameter param() { + return this.function.parameters[0]; + } + + DBSPTupleExpression body() { + return this.function.body.to(DBSPTupleExpression.class); + } + } + + /** True if the expressions compute the same value, given each map's parameter. + * The only variable an expression may reference is its map's parameter */ + static boolean equivalent(MapInfo left, DBSPExpression newLeft, MapInfo right, DBSPExpression newRight) { + return EquivalenceContext.equiv(newLeft.closure(left.param()), newRight.closure(right.param())); + } + + /** Collects the minimal expensive subexpressions of a closure's body: + * expensive nodes with no expensive proper subexpression. + * A collected expression may reference only the closure's parameter. */ + static class CollectExpensiveExpressions extends InnerVisitor { + // Minimal expensive sub-expressions: no subexpressions are expensive + final List minimal = new ArrayList<>(); + /** For each expensive node on the visit path, minimal.size() when entered */ + final List marks = new ArrayList<>(); + final DBSPParameter parameter; + /** The declaration of each variable in the closure */ + final ReferenceMap references; + + public CollectExpensiveExpressions(DBSPCompiler compiler, DBSPClosureExpression function) { + super(compiler); + this.parameter = function.parameters[0]; + ResolveReferences resolver = new ResolveReferences(compiler, false); + resolver.apply(function); + this.references = resolver.reference; + } + + /** True if every variable inside the expression resolves to a + * declaration inside it or to the closure parameter */ + boolean closed(DBSPExpression expression) { + final Set declared = new HashSet<>(); + final List used = new ArrayList<>(); + + InnerVisitor scanner = new InnerVisitor(this.compiler()) { + @Override + public VisitDecision preorder(DBSPType type) { + return VisitDecision.STOP; + } + + @Override + public void postorder(IDBSPInnerNode node) { + if (node.is(IDBSPDeclaration.class)) + declared.add(node.to(IDBSPDeclaration.class)); + } + + @Override + public void postorder(DBSPVariablePath variable) { + used.add(variable); + } + }; + scanner.apply(expression); + for (DBSPVariablePath variable : used) { + IDBSPDeclaration declaration = this.references.get(variable); + Utilities.enforce(declaration != null, + () -> "Variable " + variable + " has no declaration"); + if (declaration != this.parameter && !declared.contains(declaration)) + return false; + } + return true; + } + + @Override + public VisitDecision preorder(DBSPType type) { + return VisitDecision.STOP; + } + + @Override + // Do not look inside nested closures + public VisitDecision preorder(DBSPClosureExpression closure) { + return VisitDecision.STOP; + } + + @Override + public VisitDecision preorder(DBSPForExpression expression) { + // Statement-like, never a shareable value + return VisitDecision.STOP; + } + + @Override + public VisitDecision preorder(DBSPExpression expression) { + if (!Expensive.isExpensive(this.compiler(), expression)) + // If this expression is not expensive, no subexpressions can be either + return VisitDecision.STOP; + this.marks.add(this.minimal.size()); + return VisitDecision.CONTINUE; + } + + @Override + public void postorder(DBSPExpression expression) { + int mark = Utilities.removeLast(this.marks); + if (this.minimal.size() == mark && this.closed(expression)) + // expression is expensive (since preorder didn't stop), + // no subexpression was collected (otherwise minimal would be + // longer), and it only uses the parameter: minimal. + // A rejected open fragment leaves the mark untouched, so its + // nearest closed expensive ancestor is collected instead. + this.minimal.add(expression); + } + } + + /** Groups maps by their input port and finds clusters worth fusing: + * siblings that share an expensive subexpression. */ + static class FindFusableMaps extends CircuitVisitor { + final ContainsNow containsNow; + final Map> groups = new LinkedHashMap<>(); + /** The cluster each fused map belongs to, keyed by cluster member. + * A single-member cluster computes an expensive expression twice. */ + final Map> clusters = new HashMap<>(); + + public FindFusableMaps(DBSPCompiler compiler) { + super(compiler); + this.containsNow = new ContainsNow(compiler, true); + } + + @Override + public void postorder(DBSPMapOperator operator) { + if (!operator.getFunction().is(DBSPClosureExpression.class)) + return; + DBSPClosureExpression function = operator.getClosureFunction(); + this.containsNow.apply(function); + if (this.containsNow.found) + return; + if (!Expensive.isExpensive(this.compiler(), function)) + return; + Simplify simplify = new Simplify(this.compiler()); + function = simplify.apply(function).to(DBSPClosureExpression.class); + if (!function.body.is(DBSPTupleExpression.class)) + return; + CollectExpensiveExpressions collector = + new CollectExpensiveExpressions(this.compiler(), function); + collector.apply(function.body); + if (collector.minimal.isEmpty()) + return; + MapInfo info = new MapInfo(operator, function, collector.minimal); + this.groups.computeIfAbsent(operator.input(), p -> new ArrayList<>()).add(info); + } + + /** A fingerprint that equivalent expressions share: the printout of + * the canonical form of the expression closed over its parameter. + * Comparing fingerprints for equality is a fast equivalence check for + * closed closures. */ + String fingerprint(MapInfo info, DBSPExpression expression) { + CanonicalForm canonical = new CanonicalForm(this.compiler()); + return canonical.apply(expression.closure(info.param())).toString(); + } + + boolean hasDuplicatedExpensiveField(MapInfo info) { + DBSPExpression[] fields = Objects.requireNonNull(info.body().fields); + for (int i = 0; i < fields.length; i++) { + if (!Expensive.isExpensive(this.compiler(), fields[i])) + continue; + for (int j = 0; j < i; j++) + if (equivalent(info, fields[j], info, fields[i])) + return true; + } + return false; + } + + /** The root of the cluster that map 'i' belongs to in the union-find forest */ + int find(int[] parent, int i) { + while (parent[i] != i) + i = parent[i]; + return i; + } + + @Override + public void endVisit() { + for (List group : this.groups.values()) { + int n = group.size(); + // Union-find forest over the group: parent[i] points towards + // the root of i's cluster, and maps with one root fuse + // together. + int[] parent = new int[n]; + for (int i = 0; i < n; i++) + parent[i] = i; + // Maps sharing a fingerprint are joined; linear in the total + // number of collected expressions, instead of comparing all + // expressions of all pairs of maps + Map firstWithPrint = new HashMap<>(); + for (int i = 0; i < n; i++) { + MapInfo info = group.get(i); + Set prints = new HashSet<>(); + for (DBSPExpression expression : info.minimal()) + prints.add(this.fingerprint(info, expression)); + for (String print : prints) { + Integer first = firstWithPrint.putIfAbsent(print, i); + if (first == null) + continue; + int ri = this.find(parent, first); + int rj = this.find(parent, i); + if (ri != rj) + parent[rj] = ri; + } + } + + Map> components = new LinkedHashMap<>(); + for (int i = 0; i < n; i++) + components.computeIfAbsent(this.find(parent, i), r -> new ArrayList<>()) + .add(group.get(i)); + for (List members : components.values()) { + if (members.size() == 1 && + !this.hasDuplicatedExpensiveField(members.get(0))) + // "Fusing" a single map when it contains a repeated computation + continue; + for (MapInfo member : members) + Utilities.putNew(this.clusters, member.operator(), members); + } + } + super.endVisit(); + } + } + + /** Rewrites the clusters found by {@link FindFusableMaps} */ + static class Fuse extends CircuitCloneVisitor { + final FindFusableMaps found; + + Fuse(DBSPCompiler compiler, FindFusableMaps found) { + super(compiler, false); + this.found = found; + } + + /** Index of an equivalent column, or -1 */ + static int indexOf(List columns, DBSPExpression expression, DBSPVariablePath var) { + for (int i = 0; i < columns.size(); i++) + if (EquivalenceContext.equiv(columns.get(i).closure(var), expression.closure(var))) + return i; + return -1; + } + + /** Replace a cluster of sibling maps with one fused map computing the + * distinct fields of all members, followed by one projection per member. */ + void fuse(List cluster) { + MapInfo first = cluster.get(0); + OutputPort source = this.mapped(first.operator().input()); + DBSPVariablePath var = first.param().getType().var(); + + List columns = new ArrayList<>(); + List memberColumns = new ArrayList<>(); + for (MapInfo member : cluster) { + DBSPExpression body = member.function().call(var).reduce(this.compiler()); + Utilities.enforce(body.is(DBSPTupleExpression.class), + () -> "Fused map body is not a tuple: " + body); + DBSPExpression[] fields = Objects.requireNonNull( + body.to(DBSPTupleExpression.class).fields); + int[] cols = new int[fields.length]; + for (int i = 0; i < fields.length; i++) { + int col = indexOf(columns, fields[i], var); + if (col < 0) { + col = columns.size(); + columns.add(fields[i]); + } + cols[i] = col; + } + memberColumns.add(cols); + } + + DBSPTupleExpression fusedTuple = + new DBSPTupleExpression(columns.toArray(new DBSPExpression[0])); + DBSPMapOperator fused = new DBSPMapOperator( + first.operator().getRelNode(), fusedTuple.closure(var), source); + this.addOperator(fused); + + for (int m = 0; m < cluster.size(); m++) { + MapInfo member = cluster.get(m); + int[] cols = memberColumns.get(m); + DBSPVariablePath row = fusedTuple.getType().ref().var(); + DBSPExpression[] fields = new DBSPExpression[cols.length]; + for (int i = 0; i < cols.length; i++) + fields[i] = row.deref().field(cols[i]).applyCloneIfNeeded(); + DBSPTypeTuple outputType = + member.operator().getOutputZSetElementType().to(DBSPTypeTuple.class); + DBSPClosureExpression projection = new DBSPTupleExpression( + member.operator().getNode(), outputType, fields).closure(row); + DBSPMapOperator projected = new DBSPMapOperator( + member.operator().getRelNode(), projection, fused.outputPort()) + .addAnnotation(new IsProjection(columns.size()), DBSPMapOperator.class); + this.map(member.operator().outputPort(), projected.outputPort(), true); + } + } + + @Override + public void postorder(DBSPMapOperator operator) { + if (this.remap.containsKey(operator.outputPort())) + // Fused when the first member of its cluster was visited + return; + List cluster = this.found.clusters.get(operator); + if (cluster != null) { + this.fuse(cluster); + return; + } + super.postorder(operator); + } + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPClosureExpression.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPClosureExpression.java index 4077b43e649..93650c341ac 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPClosureExpression.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/ir/expression/DBSPClosureExpression.java @@ -43,7 +43,9 @@ import org.dbsp.util.Linq; import org.dbsp.util.Maybe; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.dbsp.util.Maybe.*; @@ -202,6 +204,14 @@ public boolean shouldInlineComposition(DBSPCompiler compiler, DBSPClosureExpress Projection projection = new Projection(compiler, true, true); projection.apply(this); if (projection.isProjection && before.body.is(DBSPBaseTupleExpression.class)) { + DBSPExpression[] fields = before.body.to(DBSPBaseTupleExpression.class).fields; + if (projection.hasIoMap() && this.parameters.length == 1 && fields != null) { + // Do not inline a projection that duplicates an expensive field + Set seen = new HashSet<>(); + for (int field : projection.getIoMap().getFieldsOfInput(0)) + if (!seen.add(field) && Expensive.isExpensive(compiler, fields[field])) + return false; + } return true; } else { int refCount = this.parameterReferences(compiler, this.parameters[0]); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Logger.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Logger.java index ac011f6f79a..c1802d53b04 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Logger.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/util/Logger.java @@ -92,7 +92,8 @@ public int setLoggingLevel(Class clazz, int level) { "visitors.outer.monotonicity", "visitors.unusedFields", "frontend", - "frontend.calciteCompiler" + "frontend.calciteCompiler", + "frontend.calciteCompiler.optimizer" }; Class locateClass(String className) { diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CollectExpensiveExpressionsTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CollectExpensiveExpressionsTests.java new file mode 100644 index 00000000000..7ec3921e4e8 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/CollectExpensiveExpressionsTests.java @@ -0,0 +1,217 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.compiler.sql.tools.BaseSQLTests; +import org.dbsp.sqlCompiler.compiler.sql.tools.ExpressionBuilder; +import org.dbsp.sqlCompiler.compiler.visitors.outer.FuseExpensiveMaps.CollectExpensiveExpressions; +import org.dbsp.sqlCompiler.compiler.visitors.outer.FuseExpensiveMaps.MapInfo; +import org.dbsp.sqlCompiler.ir.expression.DBSPClosureExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPVariablePath; +import org.dbsp.sqlCompiler.ir.type.derived.DBSPTypeTuple; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** Tests for {@link FuseExpensiveMaps.CollectExpensiveExpressions} and for the + * equivalence check {@link FuseExpensiveMaps} runs on the collected expressions. + * Each test shows the analyzed function in a comment, as toString() prints it. */ +public class CollectExpensiveExpressionsTests extends BaseSQLTests { + final ExpressionBuilder b = new ExpressionBuilder(); + final DBSPTypeTuple pair = this.b.tup(this.b.i32(), this.b.i32()); + + /** Collect the minimal expensive expressions of |param| expression */ + List collect(DBSPExpression expression, DBSPVariablePath param) { + DBSPClosureExpression closure = expression.closure(param); + CollectExpensiveExpressions collector = + new CollectExpensiveExpressions(this.testCompiler(), closure); + collector.apply(closure.body); + return collector.minimal; + } + + /** Only the call is minimal, not the addition containing it */ + @Test + public void testCall() { + // |t_0: &Tup2| f(*t_0.0) + 1 + // Extracted: f(*t_0.0) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression call = b.call("f", b.field(t, 0)); + Assert.assertEquals(List.of(call), this.collect(b.add(call, b.lit(1)), t)); + } + + /** Of two nested calls only the inner one is minimal */ + @Test + public void testNestedCalls() { + // |t_0: &Tup2| f(g(*t_0.0)) + // Extracted: g(*t_0.0) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression inner = b.call("g", b.field(t, 0)); + Assert.assertEquals(List.of(inner), this.collect(b.call("f", inner), t)); + } + + @Test + public void testCheap() { + // |t_0: &Tup2| *t_0.0 + *t_0.1 + // Extracted: nothing + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression expression = b.add(b.field(t, 0), b.field(t, 1)); + Assert.assertEquals(List.of(), this.collect(expression, t)); + } + + /** Unrelated calls are both minimal */ + @Test + public void testTwoCalls() { + // |t_0: &Tup2| f(*t_0.0) + g(*t_0.1) + // Extracted: f(*t_0.0), g(*t_0.1) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression f = b.call("f", b.field(t, 0)); + DBSPExpression g = b.call("g", b.field(t, 1)); + Assert.assertEquals(List.of(f, g), this.collect(b.add(f, g), t)); + } + + /** The same node used twice is collected once per occurrence: node sharing + * in the IR does not mean the generated code evaluates it once. */ + @Test + public void testSharedNode() { + // |t_0: &Tup2| Tup2::new(f(*t_0.0), f(*t_0.0), ) + // Extracted: f(*t_0.0), f(*t_0.0) -- the same node, twice + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression call = b.call("f", b.field(t, 0)); + Assert.assertEquals(List.of(call, call), this.collect(b.tuple(call, call), t)); + } + + /** A call whose argument is a lambda is minimal as a whole: a value under + * the lambda is computed per lambda invocation, not per row. */ + @Test + public void testLambdaOpaque() { + // |t_0: &Tup2| map_array(*t_0.0, |t_1: i32| h(t_1)) + // Extracted: map_array(*t_0.0, |t_1: i32| h(t_1)) -- the whole call + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression outer = b.call("map_array", + b.field(t, 0), b.lambda(b.i32(), a -> b.call("h", a))); + Assert.assertEquals(List.of(outer), this.collect(outer, t)); + } + + /** The collector recurses into a let expression; the initializer only + * uses the parameter and is collected, while the consumer fragment + * referencing the let variable is not comparable out of context. */ + @Test + public void testLetTransparent() { + // |t_0: &Tup2| { let t_1 = f(*t_0.0); g(t_1) } + // Extracted: f(*t_0.0) -- g(t_1) is rejected, it uses t_1 + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression initializer = b.call("f", b.field(t, 0)); + DBSPExpression let = b.let(initializer, x -> b.call("g", x)); + Assert.assertEquals(List.of(initializer), this.collect(let, t)); + } + + /** A closed fragment next to an open one is still collected */ + @Test + public void testLetSibling() { + // |t_0: &Tup2| { + // let t_1 = f(*t_0.0); + // g(t_1) + h(*t_0.1) + // } + // Extracted: f(*t_0.0), h(*t_0.1) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression initializer = b.call("f", b.field(t, 0)); + DBSPExpression closed = b.call("h", b.field(t, 1)); + DBSPExpression let = b.let(initializer, x -> b.add(b.call("g", x), closed)); + Assert.assertEquals(List.of(initializer, closed), this.collect(let, t)); + } + + /** A variable reference is open in every scope below its declaration */ + @Test + public void testNestedLet() { + // |t_0: &Tup2| { + // let t_1 = f(*t_0.0); + // { + // let t_2 = g(t_1); + // h(t_2) + // }} + // Extracted: f(*t_0.0) -- everything else uses t_1 or t_2 + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression initializer = b.call("f", b.field(t, 0)); + DBSPExpression outer = b.let(initializer, + x -> b.let(b.call("g", x), + y -> b.call("h", y))); + Assert.assertEquals(List.of(initializer), this.collect(outer, t)); + } + + /** When every expensive fragment references the let variable, the whole + * let expression is the minimal closed unit. */ + @Test + public void testLetWholesale() { + // |t_0: &Tup2| { + // let t_1 = (*t_0.0 + 1); + // g(t_1) + // } + // Extracted: the whole let expression + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression let = b.let(b.add(b.field(t, 0), b.lit(1)), x -> b.call("g", x)); + Assert.assertEquals(List.of(let), this.collect(let, t)); + } + + /** The collector recurses into block expressions the same way */ + @Test + public void testBlockTransparent() { + // |t_0: &Tup2| { + // let t_1: i32 = f(*t_0.0); + // (t_1 + 1) + // } + // Extracted: f(*t_0.0) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression initializer = b.call("f", b.field(t, 0)); + DBSPExpression block = b.block(initializer, s -> b.add(s, b.lit(1))); + Assert.assertEquals(List.of(initializer), this.collect(block, t)); + } + + /** The equivalence check must work on every pair of collected expressions. */ + @Test + public void testLetEquivalence() { + // |t_0: &Tup2| { + // let t_1 = f(*t_0.0); + // g(t_1) + // } + // Extracted: f(*t_0.0) + DBSPVariablePath t = b.refVar(this.pair); + DBSPExpression leftLet = b.let(b.call("f", b.field(t, 0)), x -> b.call("g", x)); + MapInfo left = new MapInfo(null, + b.tuple(leftLet).closure(t), this.collect(leftLet, t)); + + // Alpha-equivalent to leftLet: + // |u_0: &Tup2| { + // let u_1 = f(*u_0.0); + // g(u_1) + // } + // Extracted: f(*u_0.0), equivalent to f(*t_0.0) + DBSPVariablePath u = b.refVar(this.pair); + DBSPExpression sameLet = b.let(b.call("f", b.field(u, 0)), y -> b.call("g", y)); + MapInfo same = new MapInfo(null, + b.tuple(sameLet).closure(u), this.collect(sameLet, u)); + + // Differs from leftLet in the initializer: + // |v_0: &Tup2| { + // let v_1 = f(*v_0.1); + // g(v_1) + // } + // Extracted: f(*v_0.1), not equivalent to f(*t_0.0) + DBSPVariablePath v = b.refVar(this.pair); + DBSPExpression otherLet = b.let(b.call("f", b.field(v, 1)), z -> b.call("g", z)); + MapInfo other = new MapInfo(null, + b.tuple(otherLet).closure(v), this.collect(otherLet, v)); + + Assert.assertTrue(anyEquivalent(left, same)); + Assert.assertFalse(anyEquivalent(left, other)); + } + + /** The comparison the fusion decision performs, via fingerprints */ + static boolean anyEquivalent(MapInfo left, MapInfo right) { + boolean result = false; + for (DBSPExpression a : left.minimal()) + for (DBSPExpression c : right.minimal()) + // Must not crash for any pair + result |= FuseExpensiveMaps.equivalent(left, a, right, c); + return result; + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMapsTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMapsTests.java new file mode 100644 index 00000000000..fad7345e3c2 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/visitors/outer/FuseExpensiveMapsTests.java @@ -0,0 +1,169 @@ +package org.dbsp.sqlCompiler.compiler.visitors.outer; + +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.TableData; +import org.dbsp.sqlCompiler.compiler.sql.tools.Change; +import org.dbsp.sqlCompiler.compiler.sql.tools.SqlIoTest; +import org.dbsp.sqlCompiler.compiler.visitors.VisitDecision; +import org.dbsp.sqlCompiler.compiler.visitors.inner.InnerVisitor; +import org.dbsp.sqlCompiler.ir.expression.DBSPApplyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPLazyExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPTupleExpression; +import org.dbsp.sqlCompiler.ir.expression.DBSPZSetExpression; +import org.dbsp.sqlCompiler.ir.expression.literal.DBSPStringLiteral; +import org.dbsp.sqlCompiler.ir.type.DBSPType; +import org.junit.Assert; +import org.junit.Test; + +/** Tests for {@link FuseExpensiveMaps} */ +public class FuseExpensiveMapsTests extends SqlIoTest { + /** Counts the calls to a function in the whole circuit. */ + static class CountCalls extends InnerVisitor { + final String namePrefix; + int count = 0; + + public CountCalls(DBSPCompiler compiler, String namePrefix) { + super(compiler); + this.namePrefix = namePrefix; + } + + @Override + public VisitDecision preorder(DBSPType type) { + return VisitDecision.STOP; + } + + @Override + public void postorder(DBSPApplyExpression node) { + String name = node.getFunctionName(); + if (name != null && name.startsWith(this.namePrefix)) + this.count++; + } + } + + static final String TABLE_AND_FUNCTION = """ + CREATE TABLE data ( + id VARCHAR NOT NULL, + sid VARCHAR NOT NULL + ); + CREATE FUNCTION expensive(x VARCHAR NOT NULL) RETURNS VARCHAR NOT NULL AS + UPPER(x) || '!'; + """; + + /** Two views over the same table calling the same expensive function */ + @Test + public void testCrossViewFusion() { + String sql = TABLE_AND_FUNCTION + """ + CREATE VIEW V1 AS SELECT id, expensive(sid) AS e FROM data; + CREATE VIEW V2 AS SELECT expensive(sid) AS e, sid FROM data;"""; + DBSPCompiler compiler = this.testCompiler(); + compiler.submitStatementsForCompilation(sql); + var ccs = this.getCCS(compiler); + + CountCalls counter = new CountCalls(compiler, "expensive"); + ccs.visit(counter.getCircuitVisitor(false)); + Assert.assertEquals(1, counter.count); + + Change input = ccs.toChange("INSERT INTO data VALUES('a', 'x');"); + Change output = new Change( + new TableData("V1", new DBSPZSetExpression(new DBSPTupleExpression( + new DBSPStringLiteral("a"), new DBSPStringLiteral("X!")))), + new TableData("V2", new DBSPZSetExpression(new DBSPTupleExpression( + new DBSPStringLiteral("X!"), new DBSPStringLiteral("x"))))); + ccs.addPair(input, output); + } + + /** expensive is top-level and nested in the two calls */ + @Test + public void testUnionSharing() { + String sql = TABLE_AND_FUNCTION + """ + CREATE VIEW both AS + SELECT id, expensive(sid) AS e FROM data + UNION ALL + SELECT sid, expensive(sid) || '?' AS e FROM data;"""; + DBSPCompiler compiler = this.testCompiler(); + compiler.submitStatementsForCompilation(sql); + var ccs = this.getCCS(compiler).withStringTrim(); + + CountCalls counter = new CountCalls(compiler, "expensive"); + ccs.visit(counter.getCircuitVisitor(false)); + Assert.assertEquals(1, counter.count); + + ccs.stepWeightOne("INSERT INTO data VALUES('a', 'x');", + """ + id | e + --------- + a | X! + x | X!?"""); + } + + /** Counts {@link DBSPLazyExpression}s in the circuit. InnerCSE creates one + * for an expensive expression that a single function computes twice, so a + * fused circuit must have none. */ + static class CountLazy extends InnerVisitor { + int count = 0; + + public CountLazy(DBSPCompiler compiler) { + super(compiler); + } + + @Override + public VisitDecision preorder(DBSPType type) { + return VisitDecision.STOP; + } + + @Override + public void postorder(DBSPLazyExpression node) { + this.count++; + } + } + + @Test + public void testWithinSingleMap() { + String sql = TABLE_AND_FUNCTION + """ + CREATE VIEW dup AS + SELECT id, expensive(sid) AS a, expensive(sid) AS b FROM data;"""; + DBSPCompiler compiler = this.testCompiler(); + compiler.submitStatementsForCompilation(sql); + var ccs = this.getCCS(compiler).withStringTrim(); + + CountCalls counter = new CountCalls(compiler, "expensive"); + ccs.visit(counter.getCircuitVisitor(false)); + Assert.assertEquals(1, counter.count); + + // If fusion worked there is no CSE-ed call + CountLazy lazy = new CountLazy(compiler); + ccs.visit(lazy.getCircuitVisitor(false)); + Assert.assertEquals(0, lazy.count); + + ccs.stepWeightOne("INSERT INTO data VALUES('a', 'x');", """ + id | a | b + ------------ + a | X! | X!"""); + } + + /** Two views projecting expensive computations of the same VARIANT column. */ + @Test + public void testVariantSharing() { + String sql = """ + CREATE TABLE data ( + id VARCHAR NOT NULL PRIMARY KEY, + properties VARIANT, + sid VARCHAR + ); + CREATE FUNCTION to_ts(d VARCHAR) RETURNS TIMESTAMP AS + PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', d); + CREATE VIEW created AS + SELECT id, TO_TS(SAFE_CAST(properties['created'] AS VARCHAR)) AS c + FROM data; + CREATE VIEW created2 AS + SELECT TO_TS(SAFE_CAST(properties['created'] AS VARCHAR)) AS c, sid + FROM data;"""; + DBSPCompiler compiler = this.testCompiler(); + compiler.submitStatementsForCompilation(sql); + var ccs = this.getCCS(compiler); + + CountCalls counter = new CountCalls(compiler, "to_ts"); + ccs.visit(counter.getCircuitVisitor(false)); + Assert.assertEquals(1, counter.count); + } +}