diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java index 59c1c4f3d97..143974ce4a2 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/CompilerOptions.java @@ -72,6 +72,11 @@ public static class Language implements IDiff, IValidate { @Parameter(names = "--unaryPlusNoop", description = "Compile unary plus into a no-operation; similar to sqlite") public boolean unaryPlusNoop = false; + /** Rewrite CTE queries to use local views instead. + * Hidden: intended for testing and debugging. */ + @Parameter(names = "--cteViews", hidden = true, + description = "Convert each top-level common table expression (WITH) in a view into a LOCAL VIEW") + public boolean cteViews = false; public boolean same(Language language) { // Only compare fields that matter. @@ -84,7 +89,8 @@ public boolean same(Language language) { @Override public String toString() { return "Language{" + - "\n\tgenerateInputForEveryTable=" + this.generateInputForEveryTable + + "\n\tcteViews=" + this.cteViews + + ",\n\tgenerateInputForEveryTable=" + this.generateInputForEveryTable + ",\n\tignoreOrderBy=" + this.ignoreOrderBy + ",\n\tincrementalize=" + this.incrementalize + ",\n\tlenient=" + this.lenient + diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java index 83e4747bc80..6c36a8516f5 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/DBSPCompiler.java @@ -55,6 +55,7 @@ import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; import org.dbsp.sqlCompiler.compiler.frontend.TypeCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CteToLocalViews; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ForeignKey; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ParsedStatement; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ProgramIdentifier; @@ -587,6 +588,42 @@ void emitSql(List statements) { outputStream.close(); } + /** Compile a CREATE VIEW statement. The view's top-level common table + * expressions may be converted to LOCAL VIEWs. The result contains + * one statement per converted CTE, followed by the view itself. */ + List compileCreateView(ParsedStatement node, Map lateness) { + @Nullable List parts = null; + if (this.options.languageOptions.cteViews) + parts = this.sqlToRelCompiler.hoistCtes(node); + if (parts == null) { + try { + RelStatement fe = this.sqlToRelCompiler.compileCreateView( + node, lateness, this.sources, /* retry */ true); + return fe == null ? Linq.list() : Linq.list(fe); + } catch (CteToLocalViews.Retry retry) { + Logger.INSTANCE.belowLevel(this, 1) + .append("Could not decorrelate query; retrying with CTEs as local views") + .newline(); + parts = this.sqlToRelCompiler.hoistCtes(node); + } + if (parts == null) { + // The rewrite turned out not to apply; compile again without + // retry, so the real error surfaces. + RelStatement fe = this.sqlToRelCompiler.compileCreateView( + node, lateness, this.sources, /* retry */ false); + return fe == null ? Linq.list() : Linq.list(fe); + } + } + List result = new ArrayList<>(); + ParsedStatement view = Utilities.last(parts); + for (ParsedStatement part: parts) { + // Lateness declarations apply to the original view only + Map lat = part == view ? lateness : new HashMap<>(); + result.addAll(this.compileCreateView(part, lat)); + } + return result; + } + @Nullable DBSPCircuit runAllCompilerStages() { List parsed = this.runParser(); if (this.hasErrors()) @@ -694,32 +731,35 @@ void emitSql(List statements) { if (node.statement() instanceof SqlLateness) continue; - RelStatement fe; + List compiled; if (node.statement() instanceof SqlCreateView cv) { ProgramIdentifier viewName = ProgramIdentifier.fromSqlId(cv.name); - Map late = this.viewLateness.getOrDefault(viewName, new HashMap<>()); - fe = this.sqlToRelCompiler.compileCreateView(node, late, this.sources); + Map lateness = this.viewLateness.getOrDefault(viewName, new HashMap<>()); + compiled = this.compileCreateView(node, lateness); } else { - fe = this.sqlToRelCompiler.compile(node, this.sources); + RelStatement single = this.sqlToRelCompiler.compile(node, this.sources); + compiled = single == null + // error during compilation + ? Linq.list() + : Linq.list(single); } - if (fe == null) - // error during compilation - continue; - if (fe.is(CreateViewStatement.class)) { - CreateViewStatement cv = fe.to(CreateViewStatement.class); - Utilities.putNew(this.views, cv.getName(), cv); - } else if (fe.is(CreateTableStatement.class)) { - CreateTableStatement ct = fe.to(CreateTableStatement.class); - foreignKeys.addAll(ct.foreignKeys); - } else if (fe.is(CreateIndexStatement.class)) { - CreateIndexStatement ct = fe.to(CreateIndexStatement.class); - boolean success = this.validateCreateIndex(ct); - if (!success) - return null; - Utilities.putNew(this.indexes, ct.getName(), ct); + for (RelStatement fe: compiled) { + if (fe.is(CreateViewStatement.class)) { + CreateViewStatement cv = fe.to(CreateViewStatement.class); + Utilities.putNew(this.views, cv.getName(), cv); + } else if (fe.is(CreateTableStatement.class)) { + CreateTableStatement ct = fe.to(CreateTableStatement.class); + foreignKeys.addAll(ct.foreignKeys); + } else if (fe.is(CreateIndexStatement.class)) { + CreateIndexStatement ct = fe.to(CreateIndexStatement.class); + boolean success = this.validateCreateIndex(ct); + if (!success) + return null; + Utilities.putNew(this.indexes, ct.getName(), ct); + } + this.relToDBSPCompiler.compile(fe); } - this.relToDBSPCompiler.compile(fe); } this.setErrorContext(SourcePositionRange.INVALID); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java index eb238011f13..47cc6c2f7bc 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/CalciteToDBSPCompiler.java @@ -372,6 +372,42 @@ UnimplementedException decorrelateError(CalciteObject node) { "It looks like the compiler could not decorrelate this query.", 2555, node); } + /** True if {@link #visitCorrelate} can implement this correlate + * (an INNER unnest-shaped correlate compiled into a flat_map). */ + public static boolean isImplementableCorrelate(LogicalCorrelate correlate) { + if (correlate.getJoinType() != JoinRelType.INNER) + return false; + RelNode right = correlate.getRight(); + if (right instanceof Project project) + right = project.getInput(); + else if (right instanceof Filter filter) + right = filter.getInput(); + return right instanceof Uncollect uncollect + && uncollect.getInput() instanceof LogicalProject uncollectInput + && uncollectInput.getProjects().size() == 1; + } + + /** True if the plan contains a correlate that the compiler cannot + * implement; such a correlate means that the decorrelator has failed. */ + public static boolean hasUnimplementableCorrelate(RelNode plan) { + class Finder extends RelVisitor { + boolean found = false; + + @Override + public void visit(RelNode node, int ordinal, @org.checkerframework.checker.nullness.qual.Nullable RelNode parent) { + if (node instanceof LogicalCorrelate correlate + && !isImplementableCorrelate(correlate)) { + this.found = true; + return; + } + super.visit(node, ordinal, parent); + } + } + Finder finder = new Finder(); + finder.go(plan); + return finder.found; + } + void visitCorrelate(LogicalCorrelate correlate) { /* We decorrelate queries using Calcite's optimizer, which doesn't always work. @@ -414,12 +450,16 @@ void visitCorrelate(LogicalCorrelate correlate) { DBSPTypeTuple type = this.convertType( node.getPositionRange(), correlate.getRowType(), false).to(DBSPTypeTuple.class); - if (correlate.getJoinType() != JoinRelType.INNER) - throw new UnimplementedException("LEFT JOIN UNNEST"); + if (!isImplementableCorrelate(correlate)) { + if (correlate.getJoinType() != JoinRelType.INNER) + throw new UnimplementedException("LEFT JOIN UNNEST"); + throw this.decorrelateError(node); + } this.visit(correlate.getLeft(), 0, correlate); DBSPSimpleOperator left = this.getInputAs(correlate.getLeft(), true); DBSPTypeTuple leftElementType = left.getOutputZSetElementType().to(DBSPTypeTuple.class); + // The casts below are safe: isImplementableCorrelate checked the shape. RelNode correlateRight = correlate.getRight(); Project rightProject = null; Filter rightFilter = null; @@ -430,13 +470,8 @@ void visitCorrelate(LogicalCorrelate correlate) { rightFilter = (Filter) correlateRight; correlateRight = rightFilter.getInput(); } - if (!(correlateRight instanceof Uncollect uncollect)) - throw this.decorrelateError(node); - RelNode uncollectInput = uncollect.getInput(); - if (!(uncollectInput instanceof LogicalProject project)) - throw this.decorrelateError(node); - if (project.getProjects().size() != 1) - throw this.decorrelateError(node); + Uncollect uncollect = (Uncollect) correlateRight; + LogicalProject project = (LogicalProject) uncollect.getInput(); RexNode projection = project.getProjects().get(0); DBSPVariablePath dataVar = new DBSPVariablePath(leftElementType.ref()); ExpressionCompiler eComp = new ExpressionCompiler(correlate, dataVar, this.compiler); diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CteToLocalViews.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CteToLocalViews.java new file mode 100644 index 00000000000..1ed34a0ec22 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/CteToLocalViews.java @@ -0,0 +1,237 @@ +package org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorNamespace; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.dbsp.sqlCompiler.compiler.frontend.parser.SqlCreateView; +import org.dbsp.util.Utilities; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** Converts the top-level common table expressions of a CREATE VIEW + * statement into separate LOCAL VIEW statements. + * + *

Calcite has no relational operator for a CTE: the SqlToRelConverter + * re-converts a CTE body at every reference. A query whose CTE contains + * correlations (e.g., UNNEST) and is referenced many times produces many + * correlates that the decorrelator must solve together, which often fails. + * Compiling each CTE as a LOCAL VIEW may make the plan tractable. + * + *

Only top-level WITH items are hoisted. A WITH nested inside a subquery may + * reference outer scopes so it stays inlined (it may be hoisted recursively later). + * Recursive CTE items are never hoisted. + * + *

The rewrite runs on the SqlNode representation, after validation: + * the validator's name resolution decides which FROM identifiers reference + * a CTE, so shadowing (a CTE shadowing a table, or a nested WITH rebinding + * a name) is decided by the same logic that compiles the query. */ +public class CteToLocalViews { + /** Thrown by {@code compileCreateView} to request recompilation + * with CTEs converted to local views. */ + public static class Retry extends RuntimeException {} + + final SqlToRelCompiler compiler; + + public CteToLocalViews(SqlToRelCompiler compiler) { + this.compiler = compiler; + } + + /** Name of the local view generated for a CTE. The '-' can only appear + * in quoted user identifiers, so it's not a legal unquoted identifier. */ + static String localViewName(String view, String cte) { + return view + "-cte-" + cte; + } + + /** The query's top-level WITH, if the rewrite can hoist its items. */ + @Nullable + static SqlWith hoistableWith(SqlNode query) { + if (query instanceof SqlOrderBy orderBy) + query = orderBy.query; + if (!(query instanceof SqlWith with)) + return null; + for (SqlNode node : with.withList) { + SqlWithItem item = (SqlWithItem) node; + if (item.recursive.booleanValue()) + return null; + } + return with; + } + + public static boolean canHoist(SqlCreateView cv) { + return hoistableWith(cv.query) != null; + } + + /** Replaces references to hoisted CTEs with references to the + * corresponding local views. The original CTE name is preserved as a + * relation alias, so column references in the query keep resolving. */ + static class RewriteCteReferences extends SqlShuttle { + final SqlValidator validator; + /** Maps each WITH statement to the view name to use instead. */ + final IdentityHashMap hoisted; + + RewriteCteReferences(SqlValidator validator, + IdentityHashMap hoisted) { + this.validator = validator; + this.hoisted = hoisted; + } + + /** The local view name for a FROM identifier that references a + * hoisted CTE; null for every other node. */ + @Nullable + SqlIdentifier replacement(SqlNode node) { + if (!(node instanceof SqlIdentifier id) || !id.isSimple()) + return null; + SqlValidatorNamespace ns = this.validator.getNamespace(id); + if (ns == null) + return null; + SqlValidatorNamespace resolved; + try { + resolved = ns.resolve(); + } catch (RuntimeException ignored) { + return null; + } + if (resolved.getNode() instanceof SqlWithItem item) + return this.hoisted.get(item); + return null; + } + + @Override + public @Nullable SqlNode visit(SqlCall call) { + // An aliased reference 'cte AS alias' keeps its alias. + if (call.getKind() == SqlKind.AS && call.operandCount() >= 2) { + SqlIdentifier replacement = this.replacement(call.operand(0)); + if (replacement != null) { + List operands = new ArrayList<>(call.getOperandList()); + operands.set(0, replacement); + return call.getOperator().createCall(call.getParserPosition(), operands); + } + } + return super.visit(call); + } + + @Override + public @Nullable SqlNode visit(SqlIdentifier id) { + SqlIdentifier replacement = this.replacement(id); + if (replacement == null) + return id; + SqlIdentifier alias = new SqlIdentifier(id.getSimple(), id.getParserPosition()); + return SqlStdOperatorTable.AS.createCall(id.getParserPosition(), replacement, alias); + } + } + + static SqlNode deepCopy(SqlNode node) { + SqlShuttle copier = new SqlShuttle() { + @Override + public SqlNode visit(SqlLiteral literal) { + return SqlNode.clone(literal); + } + + @Override + public SqlNode visit(SqlIdentifier id) { + return SqlNode.clone(id); + } + + @Override + public SqlNode visit(SqlDataTypeSpec type) { + return SqlNode.clone(type); + } + + @Override + public SqlNode visit(SqlDynamicParam param) { + return SqlNode.clone(param); + } + + @Override + public SqlNode visit(SqlIntervalQualifier intervalQualifier) { + return SqlNode.clone(intervalQualifier); + } + + @Override + public @Nullable SqlNode visit(SqlCall call) { + CallCopyingArgHandler argHandler = new CallCopyingArgHandler(call, true); + call.getOperator().acceptCall(this, call, false, argHandler); + return argHandler.result(); + } + }; + return Objects.requireNonNull(node.accept(copier)); + } + + /** Rewrite a CREATE VIEW statement whose query has top-level common + * table expressions into one LOCAL VIEW per CTE, followed by the view + * itself with CTE references replaced by local view references. + * Returns null if the rewrite does not apply. + * The statements returned must all be compiled, in order. */ + @Nullable + public List apply(ParsedStatement statement) { + if (!(statement.statement() instanceof SqlCreateView cv)) + return null; + if (!canHoist(cv)) + return null; + SqlNode query = this.compiler.replaceRecursiveViews(cv.query); + if (query instanceof SqlOrderBy orderBy) { + // Move the ORDER BY inside the WITH, so that the WITH stays the + // top node of the validated query. + SqlWith with = (SqlWith) orderBy.query; + query = new SqlWith(with.getParserPosition(), with.withList, + new SqlOrderBy(orderBy.getParserPosition(), with.body, + orderBy.orderList, orderBy.offset, orderBy.fetch)); + } + + // Validate with a throwaway compiler: validation mutates internal + // validator state keyed by the query's nodes, and the statements + // emitted below must look brand-new to the main validator. + SqlToRelCompiler probe = new SqlToRelCompiler(this.compiler); + SqlValidator validator = probe.getValidator(); + SqlNode validated = validator.validate(query); + if (!(validated instanceof SqlWith with)) + // Cannot happen + return null; + + IdentityHashMap hoisted = new IdentityHashMap<>(); + Set usedNames = new HashSet<>(); + for (SqlNode node : with.withList) { + SqlWithItem item = (SqlWithItem) node; + String name = localViewName(cv.name.getSimple(), item.name.getSimple()); + // SQL allows duplicate CTE names (later ones shadow earlier ones) + while (!usedNames.add(name)) + name = name + "-"; + Utilities.putNew(hoisted, item, + new SqlIdentifier(name, item.name.getParserPosition())); + } + + RewriteCteReferences rewriter = new RewriteCteReferences(validator, hoisted); + List result = new ArrayList<>(); + for (SqlNode node : with.withList) { + SqlWithItem item = (SqlWithItem) node; + SqlNode body = Objects.requireNonNull(rewriter.visitNode(item.query)); + SqlCreateView local = new SqlCreateView( + item.getParserPosition(), false, SqlCreateView.ViewKind.LOCAL, + Utilities.getExists(hoisted, item), item.columnList, null, body); + result.add(new ParsedStatement(deepCopy(local), statement.visible())); + } + SqlNode body = Objects.requireNonNull(rewriter.visitNode(with.body)); + SqlCreateView view = new SqlCreateView( + cv.getParserPosition(), cv.getReplace(), cv.viewKind, + cv.name, cv.columnList, cv.viewProperties, body); + result.add(new ParsedStatement(deepCopy(view), statement.visible())); + return result; + } +} diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java index a50eb2b2bd9..426a846a473 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteCompiler/SqlToRelCompiler.java @@ -101,6 +101,7 @@ import org.apache.calcite.sql.SqlSetOption; import org.apache.calcite.sql.SqlTypeNameSpec; import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.dialect.OracleSqlDialect; import org.apache.calcite.sql.fun.SqlDatetimeSubtractionOperator; @@ -140,6 +141,7 @@ import org.dbsp.sqlCompiler.compiler.errors.SourcePositionRange; import org.dbsp.sqlCompiler.compiler.errors.UnimplementedException; import org.dbsp.sqlCompiler.compiler.errors.UnsupportedException; +import org.dbsp.sqlCompiler.compiler.frontend.CalciteToDBSPCompiler; import org.dbsp.sqlCompiler.compiler.frontend.ExtendedSqlParserPos; import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.optimizer.CalciteOptimizer; import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; @@ -552,6 +554,14 @@ public ExtraValidation(IErrorReporter errorReporter) { @Override public Void visit(SqlCall call) { SourcePositionRange position = new SourcePositionRange(call.getParserPosition()); + if (call instanceof SqlWithItem item && item.recursive.booleanValue()) { + this.errorReporter.reportError( + new SourcePositionRange(item.name.getParserPosition()), + "Not supported", + "RECURSIVE queries in WITH are not supported; " + + "use DECLARE RECURSIVE VIEW instead. " + + "See https://docs.feldera.com/sql/recursion"); + } SqlOperator operator = call.getOperator(); if (operator instanceof SqlFunction) { for (SqlNode node : call.getOperandList()) { @@ -2232,10 +2242,30 @@ public static String toString(final RelNode rel, SqlExplainLevel detailLevel, bo return sw.toString(); } + /** If the statement is a CREATE VIEW whose query has top-level + * common table expressions, convert each into a LOCAL VIEW. + * Returns null if the rewrite does not apply. */ + @Nullable + public List hoistCtes(ParsedStatement statement) { + return new CteToLocalViews(this).apply(statement); + } + @Nullable public CreateViewStatement compileCreateView( ParsedStatement node, Map lateness, SourceFileContents sources) { + return this.compileCreateView(node, lateness, sources, false); + } + + /** Compile a CREATE VIEW statement. + * @param lateness Lateness declared for the view's columns. + * @param allowCteRetry If true and the optimized plan cannot be + * decorrelated, but the query has hoistable CTEs, + * throw {@link CteToLocalViews.Retry}. */ + @Nullable + public CreateViewStatement compileCreateView( + ParsedStatement node, Map lateness, + SourceFileContents sources, boolean allowCteRetry) { CalciteObject object = CalciteObject.create(node); SqlCreateView cv = (SqlCreateView) node.statement(); SqlNode query = cv.query; @@ -2318,6 +2348,13 @@ public CreateViewStatement compileCreateView( RelNode optimized = this.optimize(relRoot.rel, node.visible(), this.getRelBuilder()); relRoot = relRoot.withRel(optimized); + // A correlate that survives the optimizer means the decorrelator has + // failed; converting the CTEs to local views often unblocks it. + // Thrown before this compiler's state records anything about the view. + if (allowCteRetry + && CteToLocalViews.canHoist(cv) + && CalciteToDBSPCompiler.hasUnimplementableCorrelate(optimized)) + throw new CteToLocalViews.Retry(); CreateViewStatement view = new CreateViewStatement(node, viewName, columns, cv, relRoot, emitFinal, props); // From Calcite's point of view we treat this view just as another table. diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/CalciteSqlNode.java b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/CalciteSqlNode.java index 8f870ee7c5b..15714656db9 100644 --- a/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/CalciteSqlNode.java +++ b/sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/calciteObject/CalciteSqlNode.java @@ -19,8 +19,14 @@ public boolean isEmpty() { @Override public String toString() { - return this.sqlNode.toSqlString( - SqlDialect.DatabaseProduct.POSTGRESQL.getDialect(), true) + return this.sqlNode.toSqlString(c -> c + .withDialect(SqlDialect.DatabaseProduct.POSTGRESQL.getDialect()) + .withAlwaysUseParentheses(true) + .withSelectListItemsOnSeparateLines(false) + .withUpdateSetListNewline(false) + .withIndentation(0) + // Quote only identifiers that need it + .withQuoteAllIdentifiers(false)) .toString(); } } diff --git a/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/CteToLocalViewsTests.java b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/CteToLocalViewsTests.java new file mode 100644 index 00000000000..c68832e0fd9 --- /dev/null +++ b/sql-to-dbsp-compiler/SQL-compiler/src/test/java/org/dbsp/sqlCompiler/compiler/sql/simple/CteToLocalViewsTests.java @@ -0,0 +1,168 @@ +package org.dbsp.sqlCompiler.compiler.sql.simple; + +import org.apache.calcite.sql.parser.SqlParseException; +import org.dbsp.sqlCompiler.compiler.DBSPCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.CteToLocalViews; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.ParsedStatement; +import org.dbsp.sqlCompiler.compiler.frontend.calciteCompiler.SqlToRelCompiler; +import org.dbsp.sqlCompiler.compiler.frontend.calciteObject.CalciteObject; +import org.dbsp.sqlCompiler.compiler.sql.tools.CompilerCircuitStream; +import org.dbsp.sqlCompiler.compiler.sql.tools.SqlIoTest; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; + +/** Tests for the conversion of common table expressions into local views. */ +public class CteToLocalViewsTests extends SqlIoTest { + /** Compile with the "CTE to local view" rewrite forced. */ + CompilerCircuitStream ccsNoCte(String sql) { + DBSPCompiler compiler = this.testCompiler(); + compiler.options.languageOptions.cteViews = true; + compiler.submitStatementsForCompilation(sql); + return this.getCCS(compiler); + } + + @Test + public void testRewriteShape() throws SqlParseException { + DBSPCompiler compiler = this.testCompiler(); + SqlToRelCompiler sqlToRel = compiler.sqlToRelCompiler; + List statements = sqlToRel.parseStatements(""" + CREATE TABLE data(x INT); + CREATE VIEW v AS + WITH data AS (SELECT x + 1 AS x FROM data), + doubled(y) AS (SELECT x * 2 FROM data) + SELECT data.x, d.y FROM data, doubled AS d;"""); + Assert.assertEquals(2, statements.size()); + sqlToRel.compile(statements.get(0), compiler.sources); + + List parts = sqlToRel.hoistCtes(statements.get(1)); + Assert.assertNotNull(parts); + Assert.assertEquals(3, parts.size()); + + // The CTE shadows table 'data' only for the rest of the query; + // the CTE's own body still reads the real table. + Assert.assertEquals(""" + CREATE LOCAL VIEW v-cte-data AS + SELECT (data.x + 1) AS x + FROM schema.data AS data""", + CalciteObject.create(parts.get(0)).toString()); + // The second CTE reads the first one; its column list carries over. + Assert.assertEquals(""" + CREATE LOCAL VIEW v-cte-doubled (y) AS + SELECT (data.x * 2) + FROM v-cte-data AS data""", + CalciteObject.create(parts.get(1)).toString()); + // The CTE names survive as aliases, so column references still resolve. + Assert.assertEquals(""" + CREATE VIEW v AS + SELECT data.x, d.y + FROM v-cte-data AS data, + v-cte-doubled AS d""", + CalciteObject.create(parts.get(2)).toString()); + } + + @Test + public void testChainedCtesShadowingTable() { + // CTE 'data' shadows the table 'data'; 'doubled' reads the CTE. + // Expected output validated on Postgres. + var ccs = this.ccsNoCte(""" + CREATE TABLE data(x INT); + CREATE VIEW v AS + WITH data AS (SELECT x + 1 AS x FROM data), + doubled(y) AS (SELECT x * 2 FROM data) + SELECT data.x, d.y FROM data JOIN doubled AS d ON d.y = 2 * data.x;"""); + ccs.stepWeightOne("INSERT INTO data VALUES(1), (2);", """ + x | y + -------- + 2 | 4 + 3 | 6"""); + } + + @Test + public void testNestedWith() { + // The nested WITH stays inline; its body references the hoisted CTE. + // Expected output validated on Postgres. + var ccs = this.ccsNoCte(""" + CREATE TABLE t(x INT); + CREATE VIEW v AS + WITH a AS (SELECT x FROM t) + SELECT * FROM (WITH b AS (SELECT x + 1 AS x FROM a) SELECT * FROM b) AS sub;"""); + ccs.stepWeightOne("INSERT INTO t VALUES(1), (2);", """ + x + --- + 2 + 3"""); + } + + @Test + public void testCteUsedTwice() { + // Expected output validated on Postgres. + var ccs = this.ccsNoCte(""" + CREATE TABLE t(id INT, v INT); + CREATE VIEW v AS + WITH e AS (SELECT id, v FROM t) + SELECT e.id, e.v, agg.s + FROM e JOIN (SELECT id, SUM(v) AS s FROM e GROUP BY id) AS agg + ON e.id = agg.id;"""); + ccs.stepWeightOne("INSERT INTO t VALUES(1, 10), (1, 20), (2, 30);", """ + id | v | s + -------------- + 1 | 10 | 30 + 1 | 20 | 30 + 2 | 30 | 30"""); + } + + @Test + public void testAutomaticFallback() { + // A correlated subquery whose body contains an UNNEST cannot be + // decorrelated with the CTE inlined; the compiler will retry + // with the CTE as a local view. + // Expected output validated on Postgres. + var ccs = this.getCCS(""" + CREATE TABLE t(id INT, arr INT ARRAY); + CREATE VIEW v AS + WITH e AS (SELECT t.id, u.v FROM t, UNNEST(t.arr) AS u(v)) + SELECT e.id, e.v, + (SELECT COUNT(*) FROM e AS e2 WHERE e2.id = e.id) AS cnt + FROM e;"""); + ccs.stepWeightOne("INSERT INTO t VALUES(1, ARRAY[10, 20]), (2, ARRAY[30]);", """ + id | v | cnt + --------------- + 1 | 10 | 2 + 1 | 20 | 2 + 2 | 30 | 1"""); + } + + @Test + public void testRetryRequested() throws SqlParseException { + // The query of testAutomaticFallback cannot be compiled with the + // CTE inlined: compileCreateView must request the CTE rewrite. + DBSPCompiler compiler = this.testCompiler(); + SqlToRelCompiler sqlToRel = compiler.sqlToRelCompiler; + List statements = sqlToRel.parseStatements(""" + CREATE TABLE t(id INT, arr INT ARRAY); + CREATE VIEW v AS + WITH e AS (SELECT t.id, u.v FROM t, UNNEST(t.arr) AS u(v)) + SELECT e.id, e.v, + (SELECT COUNT(*) FROM e AS e2 WHERE e2.id = e.id) AS cnt + FROM e;"""); + sqlToRel.compile(statements.get(0), compiler.sources); + try { + sqlToRel.compileCreateView(statements.get(1), new HashMap<>(), compiler.sources, true); + Assert.fail("Expected a CTE rewrite request"); + } catch (CteToLocalViews.Retry ignored) {} + } + + @Test + public void testRecursiveCteRejected() { + // Recursion must be expressed with DECLARE RECURSIVE VIEW. + this.statementsFailingInCompilation(""" + CREATE TABLE t(x INT); + CREATE VIEW v AS + WITH RECURSIVE r(x) AS (SELECT x FROM t UNION ALL SELECT x + 1 FROM r WHERE x < 10) + SELECT * FROM r;""", + "use DECLARE RECURSIVE VIEW instead"); + } +}